diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..56e5817a02 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# DOCUMENTATION ------------------------------------------------------- +# Docs folder ownership +/docs/ @nextflow-io/docs diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md index 7f9a6d64e5..363896fd8b 100644 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -4,7 +4,6 @@ When submitting a Pull Request please make sure to not include in the changeset any modification in these files: * `nextflow` -* `docs/conf.py` * `modules/nf-commons/src/main/nextflow/Const.groovy` Also, please sign-off the DCO [1] to certify you are the author of the contribution diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5483a14f18..74f060ad13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,9 +39,17 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v43 with: - files_ignore: docs + files_ignore: docs/** + + - name: List all changed files + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done - name: Setup env if: steps.changed-files.outputs.any_changed == 'true' @@ -54,7 +62,7 @@ jobs: - name: Setup Java ${{ matrix.java_version }} if: steps.changed-files.outputs.any_changed == 'true' - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{matrix.java_version}} distribution: 'temurin' @@ -67,7 +75,14 @@ jobs: - name: Test if: steps.changed-files.outputs.any_changed == 'true' - run: make test + run: | + # configure test env + if [[ "$GOOGLE_SECRET" ]]; then + echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json + export GOOGLE_APPLICATION_CREDENTIALS=$PWD/google_credentials.json + fi + # run tests + make test env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -82,7 +97,7 @@ jobs: AZURE_BATCH_ACCOUNT_KEY: ${{ secrets.AZURE_BATCH_ACCOUNT_KEY }} - name: Publish tests report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: steps.changed-files.outputs.any_changed == 'true' && always() with: name: report-unit-tests-jdk-${{ matrix.java_version }} @@ -96,6 +111,7 @@ jobs: if: ${{ !contains(github.event.head_commit.message, '[ci fast]') && needs.build.outputs.any_changed == 'true' }} needs: build runs-on: ubuntu-latest + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -117,7 +133,7 @@ jobs: NXF_GITHUB_ACCESS_TOKEN: ${{ secrets.NXF_GITHUB_ACCESS_TOKEN }} - name: Setup Java ${{ matrix.java_version }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{matrix.java_version}} distribution: 'temurin' @@ -152,7 +168,7 @@ jobs: run: tar -cvf integration-tests.tar tests/checks - name: Publish tests report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: report-${{ matrix.test_mode }}-jdk-${{ matrix.java_version }} diff --git a/Makefile b/Makefile index bf55192bd3..2479a1cd19 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # -# Copyright 2013-2023, Seqera Labs +# Copyright 2013-2024, Seqera Labs # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/NOTICE b/NOTICE index 27fc4d0513..6fcf7e0b8b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,6 +1,6 @@ NEXTFLOW -Copyright 2013-2023, Seqera Labs +Copyright 2013-2024, Seqera Labs This software includes source code and libraries developed by: @@ -142,7 +142,7 @@ This software includes source code and libraries developed by: Spack https://github.com/spack/spack - Copyright 2013-2023 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. + Copyright 2013-2024 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. Licensed under Apache License, Version 2.0 Nextflow is a registered trademark of Seqera Labs, Spain. diff --git a/VERSION b/VERSION index e1086d68d2..5eca1d6fcd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.01.0-edge +24.04.1 diff --git a/build.gradle b/build.gradle index dc6afae452..dc2722d115 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,6 +69,10 @@ allprojects { java { toolchain { languageVersion = JavaLanguageVersion.of(19) + // note: the use of Java 21 causes the error "NoClassDefFoundError: java/util/SequencedCollection" + // see also + // https://aphyr.com/posts/369-classnotfoundexception-java-util-sequencedcollection + // https://www.baeldung.com/java-21-sequenced-collections } } @@ -108,15 +112,15 @@ allprojects { } // Documentation required libraries - groovyDoc 'org.fusesource.jansi:jansi:1.11' - groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.18" - groovyDoc "org.apache.groovy:groovy-ant:4.0.18" + groovyDoc 'org.fusesource.jansi:jansi:2.4.0' + groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.21" + groovyDoc "org.apache.groovy:groovy-ant:4.0.21" } test { useJUnitPlatform() } - + // this is required due to this IDEA bug // https://youtrack.jetbrains.com/issue/IDEA-129282 sourceSets { @@ -145,28 +149,27 @@ allprojects { } // Required to run tests on Java 9 and higher in compatibility mode - if (JavaVersion.current() >= JavaVersion.VERSION_1_9) { - tasks.withType(Test) { - jvmArgs ([ - '--enable-preview', - '--add-opens=java.base/java.lang=ALL-UNNAMED', - '--add-opens=java.base/java.io=ALL-UNNAMED', - '--add-opens=java.base/java.nio=ALL-UNNAMED', - '--add-opens=java.base/java.nio.file.spi=ALL-UNNAMED', - '--add-opens=java.base/java.net=ALL-UNNAMED', - '--add-opens=java.base/java.util=ALL-UNNAMED', - '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED', - '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED', - '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED', - '--add-opens=java.base/sun.nio.fs=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.http=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.https=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.ftp=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.file=ALL-UNNAMED', - '--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED', - '--add-opens=java.base/jdk.internal.vm=ALL-UNNAMED', - ]) - } + tasks.withType(Test) { + jvmArgs ([ + '-Dorg.spockframework.mock.ignoreByteBuddy=true', + '--enable-preview', + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.io=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED', + '--add-opens=java.base/java.nio.file.spi=ALL-UNNAMED', + '--add-opens=java.base/java.net=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED', + '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens=java.base/sun.nio.fs=ALL-UNNAMED', + '--add-opens=java.base/sun.net.www.protocol.http=ALL-UNNAMED', + '--add-opens=java.base/sun.net.www.protocol.https=ALL-UNNAMED', + '--add-opens=java.base/sun.net.www.protocol.ftp=ALL-UNNAMED', + '--add-opens=java.base/sun.net.www.protocol.file=ALL-UNNAMED', + '--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED', + '--add-opens=java.base/jdk.internal.vm=ALL-UNNAMED', + ]) } /** @@ -213,21 +216,13 @@ task buildInfo { doLast { timestamp=${System.currentTimeMillis()} commitId=${project.property('commitId')} """.stripIndent() - + // -- update 'nextflow' wrapper file0 = file('nextflow') src = file0.text src = src.replaceAll(/NXF_VER\=\$\{NXF_VER:-'.*'\}/, 'NXF_VER=\\${NXF_VER:-\'' + version + '\'}') file0.text = src - // -- update sphynx - def major = version.split(/\./)[0..1].join('.') - file0 = file('docs/conf.py') - src = file0.text - src = src.replaceAll(/version *= *'[0-9a-zA-Z_\-\.]+'/, "version = '$major'" as String) - src = src.replaceAll(/release *= *'[0-9a-zA-Z_\-\.]+'/, "release = '$version'" as String) - file0.text = src - // -- update dockerfile file0 = file('docker/Dockerfile') src = file0.text diff --git a/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy b/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy index b81657cc3a..34d4049c41 100644 --- a/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy +++ b/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/changelog.txt b/changelog.txt index 17300ae45c..3466beb0e6 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,149 @@ NEXTFLOW CHANGE-LOG =================== +24.04.1 - 20 May 2024 +- Fix nf-google plugin dependency [725e2860] +- Fix nf-amazon plugin dependency [c234b09f] +- Bump nf-google@1.13.2 [633989a6] +- Bump nf-amazon@2.5.2 [e31f70e0] + +24.04.0 - 20 May 2024 +- Add 'preview' to workflow runtime metadata (#4985) [935bb1e5] +- Add enabled property to output dsl (#5008) [284415b1] +- Fix Taskbar API is not supported error [0ea09ccc] +- Fix inspect should not write history entry [c713ad51] +- Fix unexpected container resolution [a5ecf8a4] +- Improve icon loading error handling [c72e16f8] +- Remove Fusion symlink resolution (#5004) [071ea74c] +- Remove `seqera` and `defaults` from Conda default channels (#5003) [ec5ebd0b] +- Use protected visibility for updateStatus method [6871ba06] +- Workflow output definition (#4784) [cf0546b1] +- Bump Fusion 2.3 (#5005) [7176c113] +- Bump groovy-console 4.0.21-patch.2 [eb97831f] +- Bump nf-amazon@2.5.1 [96ee633d] +- Bump nf-console@1.1.3 [e8359042] +- Bump nf-google@1.13.1 [5dcb4c7a] +- Bump nf-wave@1.4.2 [73c668a6] + +24.04.0-edge - 13 May 2024 +- Add Wave and Fusion info to workflow metadata (#4945) [bb7e1c8e] +- Add `k8s.cpuLimits` config option (#3027) [3c6e96d0] +- Add account config option for grid executors (#4975) [a09e37dd] +- Add git to docs deps [aa9e1273] +- Add resourceLimits directive (#2911) [7c9d965e] +- Add support for Job arrays (#3892) [ca9bc9d4] +- Add support for clusterOptions as a list of values (#4993) [dd173e33] +- Add threads dump for troubleshooting purposes [8992ebde] +- Fix Gstring casting exception when clusterOptions is a closure [74004fbd] +- Fix Missing error code when no entry is specified [f507e9a4] +- Fix NPE in LogsCheckpoint class [deb3076d] +- Fix Prevent maxForks less than zero [7676dd9c] +- Fix Use fully qualified S3 uris in error message (#4923) [f1cffd1b] +- Fix Wave container resolution with singularity and ociMode [54ad6241] +- Fix collectFile saving to GCS with sort: false (#4965) [1418553a] +- Fix console "Plugin manager not initialised" warn (#4989) [5ff44538] +- Fix console icon (#4991) [b8a23706] +- Fix docs formatting [c02e58c7] +- Fix docs snippet [6499649d] +- Fix flaky docs test (#4957) [ea8246f6] +- Fix groovy console issue (#4988) [b9bf6410] +- Fix job array docs (#4984) [6a3347ee] +- Fix missing include error message (#4981) [aad100e1] +- Fix script error text alignment (#4681) [1dc4e4e4] +- Fix security vulnerability in logback (#4947) [0ffcc4ca] +- Fix semaphore in parallel polling monitor (#4927) [5c37fcc2] +- Fix remove commented out test lines in Azure Batch Pool opts tests. (#4914) [cb607f07] +- Guarantees K8s pod name is unique on resume (#4959) [361cef84] +- Improve config resolution docs (#4950) [019eb86c] +- Improve documentation about azcopy installation requirements for custom Azure Batch worker pools (#4911) [5c410db8] +- Improve error message for image pull time-out for Singularity/Apptainer/Charliecloud (#4974) [73015fbd] +- Remove unused const [6e91285d] +- Run task finalisation asynchronously (#4890) [e0e94227] +- Strip auth secret from logs [acf63e0e] +- Update TES executor to TES API v1.1 (#4195) [7b32c2d6] +- Update aws.md to include Cluster access (#4951) [459d725b] +- Update developer diagrams (#4922) [dcff41a5] +- Update operator return types (#4976) [a614fbe7] +- Update stale documentation in overview.md (#4968) [6a58c6d7] +- Use for instead eachLine in error formatting [4a821f46] +- azure batch autopool feature more comprehensive documentation (#4941) [adbd8903] +- Bump Gradle 8.7 [8b5cf3cc] +- Bump nf-wave@1.4.1 [830b032c] +- Bump nf-tower@1.9.1 [163683c2] +- Bump nf-google@1.13.0 [6d99a22a] +- Bump nf-ga4gh@1.3.0 [89695ed3] +- Bump nf-console@1.1.2 [357b143a] +- Bump nf-amazon@2.5.0 [6c62a60a] + +24.03.0-edge - 15 Apr 2024 +- Add custom jobName for Google Batch [df40d55f] +- Add escher to name generator class [2e6496e2] +- Add retry policy to Google Batch client [c4981dcc] +- Add retry strategy for publishing (#4839) [c9c7032c] +- Add support for Azure custom startTask (#4913) [27d01e3a] +- Add task tip extension point [eadad5b8] +- Allow secrets to be used in pipeline script (#4171) [df866a24] +- Do not print a new line when stdout is empty (#4892) [658a5ec8] +- Fix Azure pool creation [2ee4d11e] +- Fix Use of secrets in the includeConfig path [00c9f226] +- Fix coloured ANSI log bug (#4898) [a04d6983] +- Fix eval output type via bash -c wrapping (#4887) [2165a14d] +- Fix exception handling in local executor [74d7d7a8] +- Fix support for GCS requester pays bucket option [d9d61cff] +- Fix test when missing Google secret [33dc3ce0] +- Improve Charliecloud support (#4879) [287471c0] +- Improve control on azcopy install (#4883) [01447d5c] +- Improve error message when Google creds file is corrupted [a550e52f] +- Improve getting started docs (#4764) [b59111b3] +- Improve retry logic for AWS Batch executor [62926c28] +- Nextflow launch script: improving search for JAVA_CMD (#4830) [ebbbe9e7] +- Publish built-in reports as Tower reports (#4760) [b710d923] +- Remove not needed dsl=2 + error in example (#4812) [7c5779d7] +- Remove unused code from AssetManager [77365165] +- Revert "Fix failing CI tests (#4861)" [7ba2e253] +- Update NameGenerator (#4907) [248201af] +- Update Platform API endpoint (#4855) [4842423a] +- Update Wave to API v1alpha2 (#4906) [9c350872] +- Update docs (#4852) [6e2d1a94] +- Updated docs on Google Cloud setup and credentials (#4896) [7e8b5e26] +- Updated dodcs note on singularity default command (#4825) [567f5334] +- Bump groovy 4.0.20 [66c1a164] +- Bump groovy 4.0.21 [9e08390b] +- Bump nf-wave@1.4.0 [fc70dc8c] +- Bump nf-tower@1.9.0 [b0c4e2c5] +- Bump nf-google@1.12.0 [6ae25fad] +- Bump nf-azure@1.6.0 [967c2ac8] +- Bump nf-amazon@2.4.2 [ddda969e] + +24.02.0-edge - 10 Mar 2024 +- Add K8s job ttlSecondsAfterFinished option (#4434) [93627be6] +- Add NXF_CACHE_DIR environment var (#4655) [4b00170a] +- Add colours to ansi logs (#4573) [5e2ce9ed] +- Add eval output type (#4493) [df978113] +- Fix Always emit publish event for symlinks on resume (#4790) [bb5c4f9d] +- Fix Do not create local plugin path in embedded mode [9d6dd6a0] +- Fix Error while publishing S3 file with blanks [b74c0227] +- Fix Missing dependency for console command [baf29110] +- Fix typo in Azure Batch docs ('Azore') (#4735) [192bf8df] +- Fix typo in error message [a7f23305] +- Remove experimental admonition for podman [17d0dced] +- Remove square brackets from job name in LSF executor (#4799) [6e0ac72d] +- Remove unneeded const [09c957fb] +- Rename Tower -> Seqera Platform in docs and log messages (#4727) [7caffef9] +- Update Azure dependencies [1bcbaf0d] +- Update copyright info [e3089f0e] +- Use alias for HistoryFile.Record [17217a1c] +- minor cli docstring fix (#4759) [ee4b4a25] +- Bump Grengine 3.0.2 [42ca2b6f] +- Bump groovy 4.0.19 [854dc1f0] +- Bump snakeyaml 2.2 [07480779] +- Bump nf-amazon@2.4.1 [0eb84071] +- Bump nf-azure@1.5.1 [d63be8c0] +- Bump nf-cloudcache@0.4.1 [57b7004e] +- Bump nf-console@1.1.1 [b7f703f5] +- Bump nf-tower@1.8.1 [b8ffb180] +- Bump nf-wave@1.3.1 [0c542eda] +- Bump amazoncorretto 17.0.10-al2023 [3e695ad9] + 24.01.0-edge - 5 Feb 2024 - Add support for custom fuse device plugin (#4612) [a1e33193] - Fix Ignore stored process message when ansi log is enabled (#4645) [f9ba47ef] @@ -28,7 +172,7 @@ NEXTFLOW CHANGE-LOG - Bump nf-cloudcache@0.4.0 [2bc698c7] - Bump nf-azure@1.5.0 [07415ce1] - Bump nf-amazon@2.4.0 [b991e14b] -- Bump Groovy 4 (#4443) [ci fast] [9d32503b] +- Bump Groovy 4 (#4443) [9d32503b] - Bump actions/checkout@v4 [d1b3195e] - Bump logback@1.4.12 + guava@33.0.0-jre [331ff425] - Bump nextflow 23.12.0-edge as min version [63e83702] diff --git a/config/codenarc/codenarc.groovy b/config/codenarc/codenarc.groovy index 366a4062e0..bd082baef6 100644 --- a/config/codenarc/codenarc.groovy +++ b/config/codenarc/codenarc.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/config/codenarc/codenarc.xml b/config/codenarc/codenarc.xml index f82c0be0bd..aeffe439b9 100644 --- a/config/codenarc/codenarc.xml +++ b/config/codenarc/codenarc.xml @@ -1,5 +1,5 @@ CacheDB : createInstance CacheDB --* CacheStore diff --git a/docs/developer/diagrams/nextflow.cloud.aws.nio.mmd b/docs/developer/diagrams/nextflow.cloud.aws.nio.mmd index 523c5f971a..69242f88b2 100644 --- a/docs/developer/diagrams/nextflow.cloud.aws.nio.mmd +++ b/docs/developer/diagrams/nextflow.cloud.aws.nio.mmd @@ -2,6 +2,8 @@ classDiagram %% %% nextflow.cloud.aws.nio %% + FileSystemProvider <|-- S3FileSystemProvider + S3FileSystemProvider --> S3FileSystem : newFileSystem class S3FileSystem { diff --git a/docs/developer/diagrams/nextflow.config.mmd b/docs/developer/diagrams/nextflow.config.mmd index 611c19a413..6962ab5b1b 100644 --- a/docs/developer/diagrams/nextflow.config.mmd +++ b/docs/developer/diagrams/nextflow.config.mmd @@ -2,9 +2,6 @@ classDiagram %% %% nextflow.config %% - CmdRun --> ConfigMap : run Session --* ConfigMap - - ConfigBuilder --> ConfigParser : build + CmdRun --> ConfigBuilder : run ConfigBuilder --> ConfigMap : build - ConfigParser --> ConfigBase : parse diff --git a/docs/developer/diagrams/nextflow.executor.mmd b/docs/developer/diagrams/nextflow.executor.mmd index 4044bbced4..08c33527c2 100644 --- a/docs/developer/diagrams/nextflow.executor.mmd +++ b/docs/developer/diagrams/nextflow.executor.mmd @@ -2,19 +2,16 @@ classDiagram %% %% nextflow.executor %% - ProcessDef --> Executor : run - %% ExecutorFactory --> Executor : getExecutor + ExecutorFactory --> Executor : getExecutor - TaskProcessor --* Executor - - %% class Executor { - %% name : String - %% monitor : TaskMonitor - %% } - %% Executor --* TaskMonitor - %% Executor --> TaskHandler : submit + class Executor { + name : String + monitor : TaskMonitor + } + Executor --* TaskMonitor + Executor --> TaskHandler : submit - %% TaskMonitor <|-- TaskPollingMonitor + TaskMonitor <|-- TaskPollingMonitor class TaskPollingMonitor { capacity : int @@ -23,17 +20,13 @@ classDiagram dumpInterval : Duration } - %% TaskPollingMonitor <|-- LocalPollingMonitor + TaskPollingMonitor <|-- LocalPollingMonitor class LocalPollingMonitor { maxCpus : int maxMemory : long } - %% class TaskHandler { - %% task : TaskRun - %% } - Executor <|-- AbstractGridExecutor Executor <|-- LocalExecutor %% Executor <|-- NopeExecutor @@ -49,19 +42,10 @@ classDiagram %% PbsExecutor <|-- PbsProExecutor %% SgeExecutor <|-- CrgExecutor - LocalExecutor --> LocalPollingMonitor : init - LocalExecutor --> LocalTaskHandler : submit - LocalExecutor --> NativeTaskHandler : submit - LocalTaskHandler --> BashWrapperBuilder : submit - - AbstractGridExecutor --> TaskPollingMonitor : init - AbstractGridExecutor --> GridTaskHandler : submit - GridTaskHandler --> BashWrapperBuilder : submit - %% TaskHandler <|-- CachedTaskHandler - %% TaskHandler <|-- GridTaskHandler - %% TaskHandler <|-- LocalTaskHandler - %% TaskHandler <|-- NativeTaskHandler + TaskHandler <|-- GridTaskHandler + TaskHandler <|-- LocalTaskHandler + TaskHandler <|-- NativeTaskHandler %% TaskHandler <|-- NopeTaskHandler %% TaskHandler <|-- StoredTaskHandler diff --git a/docs/developer/diagrams/nextflow.extension.mmd b/docs/developer/diagrams/nextflow.extension.mmd index 4cb817c2e7..290246b883 100644 --- a/docs/developer/diagrams/nextflow.extension.mmd +++ b/docs/developer/diagrams/nextflow.extension.mmd @@ -26,3 +26,5 @@ classDiagram OperatorImpl --> ToListOp : toList, toSortedList OperatorImpl --> TransposeOp : transpose OperatorImpl --> UntilOp : until + + WorkflowBinding --> OpCall : invokeMethod diff --git a/docs/developer/diagrams/nextflow.script.mmd b/docs/developer/diagrams/nextflow.script.mmd index aa6869d0cd..4b22d49cf7 100644 --- a/docs/developer/diagrams/nextflow.script.mmd +++ b/docs/developer/diagrams/nextflow.script.mmd @@ -9,7 +9,6 @@ classDiagram session : Session } ScriptRunner --* ScriptFile - ScriptRunner --* Session ScriptRunner --> ScriptParser : execute ScriptParser --> BaseScript : parse @@ -22,8 +21,15 @@ classDiagram projectName : String } - Session --* BaseScript - Session --* ScriptBinding + class BaseScript { + meta : ScriptMeta + entryFlow : WorkflowDef + } + BaseScript --* ScriptBinding + BaseScript --* ScriptMeta + BaseScript --> IncludeDef : include + + IncludeDef --> ScriptParser : load0 class ScriptBinding { scriptPath : Path @@ -33,16 +39,6 @@ classDiagram entryName : String } - IncludeDef --> BaseScript : load0 - - class BaseScript { - meta : ScriptMeta - entryFlow : WorkflowDef - } - BaseScript --* ScriptMeta - %% BaseScript --> ProcessDef : process - %% BaseScript --> WorkflowDef : workflow - class ScriptMeta { scriptPath : Path definitions : Map @@ -68,11 +64,9 @@ classDiagram baseName : String rawBody : Closure~BodyDef~ } - ProcessDef --> ProcessConfig : run - ProcessDef --> BodyDef : run - ProcessDef --> Executor : run - ProcessDef --> TaskProcessor : run - ProcessDef --> ChannelOut : run + ProcessDef --* ProcessConfig + ProcessDef --* BodyDef + ProcessDef --* ChannelOut class WorkflowDef { name : String @@ -82,8 +76,8 @@ classDiagram variableNames : Set~String~ } WorkflowDef --* BodyDef - WorkflowDef --> WorkflowBinding : run - WorkflowDef --> ChannelOut : run + WorkflowDef --* WorkflowBinding + WorkflowDef --* ChannelOut class ProcessConfig { configProperties : Map diff --git a/docs/developer/diagrams/nextflow.secret.mmd b/docs/developer/diagrams/nextflow.secret.mmd index 547ade7b93..1ee67a0363 100644 --- a/docs/developer/diagrams/nextflow.secret.mmd +++ b/docs/developer/diagrams/nextflow.secret.mmd @@ -2,7 +2,9 @@ classDiagram %% %% nextflow.secret %% - CmdRun --> SecretsProvider : run + ConfigBuilder --> SecretsLoader : build + BaseScript --> SecretsLoader : run + BashWrapperBuilder --> SecretsLoader : build SecretsLoader --> SecretsProvider : load SecretsProvider --> Secret : getSecret diff --git a/docs/developer/diagrams/nextflow.trace.mmd b/docs/developer/diagrams/nextflow.trace.mmd index 891cfcc698..9498988d4c 100644 --- a/docs/developer/diagrams/nextflow.trace.mmd +++ b/docs/developer/diagrams/nextflow.trace.mmd @@ -2,21 +2,12 @@ classDiagram %% %% nextflow.trace %% - direction LR + Session --> TraceObserverFactory : init - %% TraceObserverFactory "1" --> "*" TraceObserver : create - %% TraceObserver <|-- AnsiLogObserver - %% TraceObserver <|-- GraphObserver - %% TraceObserver <|-- ReportObserver - %% TraceObserver <|-- TimelineObserver - %% TraceObserver <|-- TraceFileObserver - %% TraceObserver <|-- WebLogObserver - %% TraceObserver <|-- WorkflowStatsObserver - - Session --> AnsiLogObserver : init - Session --> GraphObserver : init - Session --> ReportObserver : init - Session --> TimelineObserver : init - Session --> TraceFileObserver : init - Session --> WebLogObserver : init - Session --> WorkflowStatsObserver : init + TraceObserverFactory "1" --> "*" TraceObserver : create + TraceObserver <|-- AnsiLogObserver + TraceObserver <|-- GraphObserver + TraceObserver <|-- ReportObserver + TraceObserver <|-- TimelineObserver + TraceObserver <|-- TraceFileObserver + TraceObserver <|-- WorkflowStatsObserver diff --git a/docs/developer/diagrams/overview.mmd b/docs/developer/diagrams/overview.mmd new file mode 100644 index 0000000000..a59e924c47 --- /dev/null +++ b/docs/developer/diagrams/overview.mmd @@ -0,0 +1,69 @@ +flowchart TB + subgraph Launcher + subgraph CmdRun + subgraph AssetManager + ScriptFile + end + subgraph ConfigBuilder + ConfigParser([ConfigParser]) + ConfigBase([ConfigBase]) + end + subgraph ScriptRunner + subgraph Session + ConfigMap + DAG + ExecutorFactory([ExecutorFactory]) + subgraph TaskProcessor + TaskRun + end + subgraph Executor + subgraph TaskMonitor + TaskHandler + end + TaskBean + BashWrapperBuilder([BashWrapperBuilder]) + end + TraceRecord + CacheFactory([CacheFactory]) + CacheDB + TraceObserver([TraceObserver]) + end + ScriptParser([ScriptParser]) + BaseScript([BaseScript]) + subgraph ScriptMeta + WorkflowDef([WorkflowDef]) + ProcessDef([ProcessDef]) + FunctionDef([FunctionDef]) + end + IncludeDef([IncludeDef]) + OpCall([OpCall]) + end + ConfigParser --> ConfigBase + ConfigBase --> ConfigMap + ScriptFile --> ScriptParser + ScriptParser --> BaseScript + BaseScript --> WorkflowDef + BaseScript --> ProcessDef + BaseScript --> FunctionDef + BaseScript --> IncludeDef + IncludeDef --> ScriptParser + WorkflowDef --> OpCall + OpCall --> DAG + ProcessDef --> DAG + DAG --> TaskRun + TaskRun --> DAG + ExecutorFactory --> Executor + ConfigMap --> Executor + ProcessDef --> TaskProcessor + ConfigMap --> TaskProcessor + TaskRun --> TaskHandler + TaskRun --> TaskBean + TaskBean --> BashWrapperBuilder + BashWrapperBuilder --> TaskHandler + CacheFactory --> CacheDB + TaskHandler --> CacheDB + TaskHandler --> TraceRecord + TraceRecord --> CacheDB + TaskHandler --> TraceObserver + end + end \ No newline at end of file diff --git a/docs/developer/index.md b/docs/developer/index.md index 8d2c7d6979..48d22817ae 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -1,3 +1,5 @@ +(contributing-page)= + # Overview This section provides a high-level overview of the Nextflow source code for users who want to understand or contribute to it. Rather than a comprehensive API documentation, these docs simply provide a conceptual map to help you understand the key concepts of the Nextflow implementation, and to quickly find code sections of interest for further investigation. @@ -142,7 +144,7 @@ If you need to test changes to the `nextflow` launcher script, you can run it di ### Groovy REPL -The `groovysh` command provides a command-line REPL that you can use to play around with Groovy code independently of Nextflow. The `groovyConsole` command provides a graphical REPL similar to `nextflow console`. These commands require a standalone Groovy distribution, which can be installed as described for Java in {ref}`Getting started `. +The `groovysh` command provides a command-line REPL that you can use to play around with Groovy code independently of Nextflow. The `groovyConsole` command provides a graphical REPL similar to `nextflow console`. These commands require a standalone Groovy distribution, which can be installed as described for Java in {ref}`Getting started `. :::{note} If you are using WSL, you must also install an X server for Windows, such as [VcXsrv](https://sourceforge.net/projects/vcxsrv/) or [Xming](http://www.straightrunning.com/XmingNotes/), in order to use these commands. diff --git a/docs/developer/nextflow.ast.md b/docs/developer/nextflow.ast.md index e6c99f5bec..5048ca6a8d 100644 --- a/docs/developer/nextflow.ast.md +++ b/docs/developer/nextflow.ast.md @@ -23,7 +23,7 @@ You can see the effect of Nextflow's AST transforms by using the Nextflow consol 3. Execute the script 4. Go to **Script** > **Inspect AST** -Here is the example from {ref}`getstarted-first`: +Here is the example from {ref}`your-first-script`: ```groovy params.str = 'Hello world!' diff --git a/docs/developer/nextflow.executor.md b/docs/developer/nextflow.executor.md index 65c0771549..8be432393e 100644 --- a/docs/developer/nextflow.executor.md +++ b/docs/developer/nextflow.executor.md @@ -14,8 +14,8 @@ Some classes may be excluded from the above diagram for brevity. ## Notes -The `Executor` class is the base class for all Nextflow executors. The main purpose of an `Executor` is to submit tasks to an underlying compute environment, such as an HPC scheduler or cloud batch executor. It uses a `TaskMonitor` to manage the lifecycle of all tasks and `TaskHandler`s to manage each individual task. Most executors use the same polling monitor, but each executor implements its own task handler to customize it for a particular compute environment. See [nextflow.processor](nextflow.processor.md) for more details about these classes. +The `Executor` class is the base class for all Nextflow executors. The main purpose of an `Executor` is to submit tasks to an underlying compute environment, such as an HPC scheduler or cloud batch executor. It uses a `TaskMonitor` to manage the lifecycle of all tasks and a `TaskHandler` to manage each individual task. Most executors use the same polling monitor, but each executor implements its own task handler to customize it for a particular compute environment. See [nextflow.processor](nextflow.processor.md) for more details about these classes. -The built-in executors include the local executor (`LocalExecutor`) and the various grid executors (SLURM, PBS, LSF, etc), all of which extend `AbstractGridExecutor`. The `LocalExecutor` implements both "local" tasks (processes with a `script` or `shell` block) and "native" tasks (processes with an `exec` block). +The built-in executors include the local executor (`LocalExecutor`) and the various grid executors (SLURM, PBS, LSF, etc), all of which extend `AbstractGridExecutor`. The `LocalExecutor` implements both "script" tasks (processes with a `script` or `shell` block) and "native" tasks (processes with an `exec` block). The `BashWrapperBuilder` is used by executors to generate the wrapper script (`.command.run`) for a task, from a template script called `command-run.txt`, as well as the task configuration and the execution environment. diff --git a/docs/developer/nextflow.k8s.md b/docs/developer/nextflow.k8s.md index 99168b0d6c..1f51e0dac5 100644 --- a/docs/developer/nextflow.k8s.md +++ b/docs/developer/nextflow.k8s.md @@ -14,4 +14,4 @@ Some classes may be excluded from the above diagram for brevity. ## Notes -The Kubernetes integration uses the K8s REST API to interact with K8s clusters, and relies on the `kubectl` command and `~/.kube/config` file for authentication. +The Kubernetes integration uses the K8s HTTP API to interact with K8s clusters, and relies on the `kubectl` command and `~/.kube/config` file for authentication. diff --git a/docs/developer/nextflow.processor.md b/docs/developer/nextflow.processor.md index e58125954f..ec5e2fa0fa 100644 --- a/docs/developer/nextflow.processor.md +++ b/docs/developer/nextflow.processor.md @@ -14,8 +14,10 @@ Some classes may be excluded from the above diagram for brevity. ## Notes -While the [`executor`](nextflow.executor.md) package defines how tasks are submitted to a particular execution environment (such as an HPC scheduler), the `processor` package defines how tasks are created and executed. As such, these packages work closely together, and in fact several components of the `Executor` interface, specifically the `TaskHandler` and `TaskMonitor`, are defined in this package. +While the [`executor`](nextflow.executor.md) package defines how tasks are submitted to a particular execution backend (such as an HPC scheduler), the `processor` package defines how tasks are created and executed. As such, these packages work closely together, and in fact several components of the `Executor` interface, specifically the `TaskHandler` and `TaskMonitor`, are defined in this package. -The `TaskProcessor` is by far the largest and most complex class in this package. It implements both the dataflow operator for a given process as well as the task execution logic. In other words, it defines the mapping from an abstract process definition with concrete channel inputs into concrete task executions. +The `TaskProcessor` is by far the largest and most complex class in this package. It implements both the dataflow operator for a given process as well as the task execution logic. In other words, it defines the mapping from an abstract process definition with input and output channels into concrete task executions. A `TaskRun` represents a particular task execution. There is also `TaskBean`, which is a serializable representation of a task. Legends say that `TaskBean` was originally created to support a "daemon" mode in which Nextflow would run on both the head node and the worker nodes, so the Nextflow "head" would need to send tasks to the Nextflow "workers". This daemon mode was never completed, but echoes of it remain (see `CmdNode`, `DaemonLauncher`, and the `nf-ignite` plugin). + +When a `TaskProcessor` receives a set of input values, it creates a `TaskRun` and submits it to an `Executor`, which in turn submits the task to a underlying execution backend. The executor's `TaskMonitor` then monitors the status of the task, and when it is completed, returns it to the task processor for finalization. If the task completed successfully, the task processor collects the task outputs and emits them on the corresponding output channels. If the task failed, the task processor will retry it if possible, or else return a task error to the workflow run. diff --git a/docs/developer/nextflow.script.md b/docs/developer/nextflow.script.md index 2757987c3f..7c76f1c2bd 100644 --- a/docs/developer/nextflow.script.md +++ b/docs/developer/nextflow.script.md @@ -14,7 +14,7 @@ Some classes may be excluded from the above diagram for brevity. ## Notes -The execution of a Nextflow pipeline occurs in two phases. In the first phase, Nextflow parses and runs the script (using the language extensions in [nextflow.ast](nextflow.ast.md) and [nextflow.extension](nextflow.extension.md)), which constructs the workflow DAG. In the second phase, Nextflow executes the workflow. +The execution of a Nextflow pipeline occurs in two phases. In the first phase, Nextflow parses and runs the script (using the language extensions in [nextflow.ast](nextflow.ast.md) and [nextflow.extension](nextflow.extension.md)), which produces the workflow DAG. In the second phase, Nextflow executes the workflow. ```{note} In DSL1, there was no separation between workflow construction and execution -- dataflow operators were executed as soon as they were constructed. DSL2 introduced lazy execution in order to separate process definition from execution, and thereby facilitate subworkflows and modules. diff --git a/docs/developer/nextflow.trace.md b/docs/developer/nextflow.trace.md index ae90c9be4d..8e86092c6e 100644 --- a/docs/developer/nextflow.trace.md +++ b/docs/developer/nextflow.trace.md @@ -14,4 +14,4 @@ Some classes may be excluded from the above diagram for brevity. ## Notes -The `TraceObserver` interface defines a set of hooks into the workflow execution, such as when a workflow starts and completes, when a task starts and completes, and when an output file is published. The `Session` maintains a list of all observers and triggers each hook when the corresponding event occurs. Implementing classes can use these hooks to perform custom behaviors. In fact, this interface is used to implemented several core features, including the various execution reports, DAG renderer, and the integration with Nextflow Tower. +The `TraceObserver` interface defines a set of hooks into the workflow execution, such as when a workflow starts and completes, when a task starts and completes, and when an output file is published. The `Session` maintains a list of all observers and triggers each hook when the corresponding event occurs. Implementing classes can use these hooks to perform custom behaviors. In fact, this interface is used to implement several core features, including the various execution reports, DAG renderer, and the integration with Seqera Platform. diff --git a/docs/dsl1.md b/docs/dsl1.md index 5635590b3d..50cea72d4e 100644 --- a/docs/dsl1.md +++ b/docs/dsl1.md @@ -18,7 +18,7 @@ export NXF_DEFAULT_DSL=2 ## Processes and workflows -In DSL1, a process definition is also the process invocation. Process inputs and outputs are connected to channels using `from` and `into`. Here is the {ref}`getstarted-first` example written in DSL1: +In DSL1, a process definition is also the process invocation. Process inputs and outputs are connected to channels using `from` and `into`. Here is the {ref}`your-first-script` example written in DSL1: ```groovy nextflow.enable.dsl=1 diff --git a/docs/executor.md b/docs/executor.md index f623d56892..53b11eea11 100644 --- a/docs/executor.md +++ b/docs/executor.md @@ -121,56 +121,61 @@ By default, Flux will send all output to the `.command.log` file. To send this o :::{warning} *Experimental: may change in a future release.* ::: +:::{versionchanged} 23.07.0-edge +Support for automatic upload of the `bin` directory was added. +::: + +:::{versionchanged} 24.04.0 +Support for process output directories and output globs was added. +::: + The [Task Execution Schema](https://github.com/ga4gh/task-execution-schemas) (TES) project by the [GA4GH](https://www.ga4gh.org) standardization initiative is an effort to define a standardized schema and API for describing batch execution tasks in a portable manner. Nextflow supports the TES API via the `tes` executor, which allows the submission of workflow tasks to a remote execution backend exposing a TES API endpoint. -To use this feature, define the following variables in the workflow launching environment: +The pipeline processes must specify the Docker image to use by defining the `container` directive, either in the pipeline script or the `nextflow.config` file. Additionally, the pipeline work directory must be accessible to the TES backend. -```bash -export NXF_MODE=ga4gh -export NXF_EXECUTOR=tes -export NXF_EXECUTOR_TES_ENDPOINT='http://back.end.com' +To enable this executor, add the following settings to your Nextflow configuration: + +```groovy +plugins { + id 'nf-ga4gh' +} + +process.executor = 'tes' +tes.endpoint = '' ``` -It is important that the endpoint is specified without the trailing slash; otherwise, the resulting URLs will not be normalized and the requests to TES will fail. +The default endpoint is `http://localhost:8000`. It is important that the endpoint is specified without the trailing slash; otherwise, the resulting URLs will not be normalized and the requests to TES will fail. -You will then be able to run your workflow over TES using the usual Nextflow command line. Be sure to specify the Docker image to use, i.e.: +The TES API supports multiple forms of authentication: -```bash -nextflow run rnaseq-nf -with-docker alpine -``` +```groovy +// basic +tes.basicUsername = '' +tes.basicPassword = '' -:::{note} -If the variable `NXF_EXECUTOR_TES_ENDPOINT` is omitted, the default endpoint is `http://localhost:8000`. -::: +// API key +tes.apiKeyParamMode = '' // 'query' or 'header' +tes.apiKeyParamName = '' +tes.apiKey = '' + +// OAuth +tes.oauthToken = '' +``` :::{tip} -You can use a local [Funnel](https://ohsu-comp-bio.github.io/funnel/) server using the following launch command line: +You can deploy a local [Funnel](https://ohsu-comp-bio.github.io/funnel/) server using the following command: ```bash ./funnel server --Server.HTTPPort 8000 --LocalStorage.AllowedDirs $HOME run ``` - -(tested with version 0.8.0 on macOS) ::: -:::{warning} -Make sure the TES backend can access the Nextflow work directory when data is exchanged using a local or shared file system. +:::{note} +While the TES API is designed to abstract workflow managers from direct storage access, Nextflow still needs to access the shared work directory used by your TES endpoint. For example, if your TES endpoint is located in Azure and uses Azure Blob storage to store the work directory, you still need to provide the necessary Azure credentials for Nextflow to access the Blob storage. ::: -### Known Limitations - -- Automatic deployment of workflow scripts in the `bin` folder is not supported. - - :::{versionchanged} 23.07.0-edge - Automatic upload of the `bin` directory is now supported. - ::: - -- Process output directories are not supported. For details see [#76](https://github.com/ga4gh/task-execution-schemas/issues/76). - -- Glob patterns in process output declarations are not supported. For details see [#77](https://github.com/ga4gh/task-execution-schemas/issues/77). - (google-batch-executor)= ## Google Cloud Batch @@ -280,30 +285,6 @@ Resource requests and other job characteristics can be controlled via the follow - {ref}`process-memory` - {ref}`process-time` -(ignite-executor)= - -## Ignite - -:::{warning} -This feature is no longer maintained. -::: - -:::{versionchanged} 22.01.0-edge -The `ignite` executor must be enabled via the `nf-ignite` plugin. -::: - -The `ignite` executor allows you to run a pipeline on an [Apache Ignite](https://ignite.apache.org/) cluster. - -To enable this executor, set `process.executor = 'ignite'` in the `nextflow.config` file. - -Resource requests and other job characteristics can be controlled via the following process directives: - -- {ref}`process-cpus` -- {ref}`process-disk` -- {ref}`process-memory` - -See the {ref}`ignite-page` page to learn how to configure Nextflow to deploy and run an Ignite cluster in your infrastructure. - (k8s-executor)= ## Kubernetes @@ -431,12 +412,20 @@ Resource requests and other job characteristics can be controlled via the follow - {ref}`process-queue` - {ref}`process-time` -### Known Limitations +When specifying `clusterOptions` as a string, multiple options must be separated by semicolons to ensure that the job script is formatted correctly: +```groovy +clusterOptions = '-t besteffort;--project myproject' +``` + +:::{versionadded} 24.04.0 +::: + +The same behavior can now be achieved using a string list: +```groovy +clusterOptions = [ '-t besteffort', '--project myproject' ] +``` -- Multiple `clusterOptions` should be semicolon-separated to ensure that the OAR job script is accurately formatted: - ```groovy - clusterOptions = '-t besteffort;--project myproject' - ``` +See {ref}`process-clusteroptions` for details. (pbs-executor)= diff --git a/docs/fusion.md b/docs/fusion.md index 35e54abf24..a10e7ab7cb 100644 --- a/docs/fusion.md +++ b/docs/fusion.md @@ -179,39 +179,4 @@ process.scratch = false ## Advanced settings -The following configuration options are available: - -`fusion.enabled` -: Enable/disable the use of Fusion file system. - -`fusion.cacheSize` -: :::{versionadded} 23.11.0-edge - ::: -: The maximum size of the local cache used by the Fusion client. - -`fusion.containerConfigUrl` -: The URL from where the container layer provisioning the Fusion client is downloaded. - -`fusion.exportStorageCredentials` -: :::{versionadded} 23.05.0-edge - This option was previously named `fusion.exportAwsAccessKeys`. - ::: -: When `true` the access credentials required by the underlying object storage are exported the pipeline jobs execution environment. - -`fusion.logLevel` -: The level of logging emitted by the Fusion client. - -`fusion.logOutput` -: Where the logging output is written. - -`fusion.privileged` -: :::{versionadded} 23.10.0 - ::: -: This allows disabling the privileged container execution when using the Fusion file system. - The effective use of this setting depends on the target execution. Currently, it's only supported by the Kubernetes - executor which requires the use the [k8s-fuse-plugin](https://github.com/nextflow-io/k8s-fuse-plugin) to be installed - in the target cluster (default: `true`). - -`fusion.tags` -: The pattern that determines how tags are applied to files created via the Fusion client. To disable tags - set it to `false`. (default: `[.command.*|.exitcode|.fusion.*](nextflow.io/metadata=true),[*](nextflow.io/temporary=true)`) +Fusion advanced configuration settings are described in the {ref}`Fusion ` section on the Nextflow configuration page. diff --git a/docs/getstarted.md b/docs/getstarted.md deleted file mode 100644 index 2b182f28c4..0000000000 --- a/docs/getstarted.md +++ /dev/null @@ -1,211 +0,0 @@ -(getstarted-page)= - -# Getting started - -(getstarted-requirement)= - -## Requirements - -Nextflow can be used on any POSIX compatible system (Linux, macOS, etc). It requires Bash 3.2 (or later) and [Java 11 (or later, up to 21)](http://www.oracle.com/technetwork/java/javase/downloads/index.html) to be installed. - -For the execution in a cluster of computers, the use of a shared file system is required to allow the sharing of tasks input/output files. - -Nextflow can also be run on Windows through [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux). - -:::{tip} -We recommend that you install Java through [SDKMAN!](https://sdkman.io/), and that you use the latest LTS version of Corretto or Temurin. See [this website](https://whichjdk.com/) for more information. While other Java distros may work at first or even most of the time, many users have experienced issues that are difficult to debug and are usually resolved by using one of the recommended distros. - -To install Corretto 17: - -```bash -sdk install java 17.0.6-amzn -``` - -To install Temurin 17: - -```bash -sdk install java 17.0.6-tem -``` -::: - -(getstarted-install)= - -## Installation - -Nextflow is distributed as a self-installing package, which means that it does not require any special installation procedure. - -It only needs two easy steps: - -1. Download the executable package by copying and pasting either one of the following commands in your terminal window: `wget -qO- https://get.nextflow.io | bash` - - Or, if you prefer `curl`: `curl -s https://get.nextflow.io | bash` - - This will create the `nextflow` main executable file in the current directory. - -2. Make the binary executable on your system by running `chmod +x nextflow`. - -3. Optionally, move the `nextflow` file to a directory accessible by your `$PATH` variable (this is only required to avoid remembering and typing the full path to `nextflow` each time you need to run it). - -:::{tip} -Set `export CAPSULE_LOG=none` to make the dependency installation logs less verbose. -::: - -:::{tip} -If you don't have `curl` or `wget`, you can also download the Nextflow launcher script from the [project releases page](https://github.com/nextflow-io/nextflow/releases/latest) on GitHub, in lieu of step 1. -::: - -:::{tip} -To avoid downloading the dependencies, you can also use the `nextflow-VERSION-all` distribution available for every Nextflow release on Github. - -1. Go to the [Github releases page](https://github.com/nextflow-io/nextflow/releases) and expand the `Assets` section for a specific release. -2. Copy the URL of the `nextflow-VERSION-all` asset and enter the download command in your terminal, e.g. `wget -qO- ASSET-URL`. It will create the completely self-contained `nextflow-VERSION-all` executable file in the current directory. -::: - -## Updates - -Having Nextflow installed in your computer you can update to the latest version using the following command: - -```bash -nextflow self-update -``` - -:::{tip} -You can temporarily switch to a specific version of Nextflow by prefixing the `nextflow` command with the `NXF_VER` environment variable. For example: - -```bash -NXF_VER=20.04.0 nextflow run hello -``` -::: - -## Stable and Edge releases - -A *stable* version of Nextflow is released on a six-months basic schedule, in the 1st and 3rd quarter of every year. - -Along with the stable release, an *edge* version is released on a monthly basis. This version is useful to test and use most recent updates and experimental features. - -To use the latest edge release run the following snippet in your shell terminal: - -```bash -export NXF_EDGE=1 -nextflow self-update -``` - -(getstarted-first)= - -## Your first script - -Copy the following example into your favorite text editor and save it to a file named `tutorial.nf`: - -```{literalinclude} snippets/your-first-script.nf -:language: groovy -``` - -:::{note} -For versions of Nextflow prior to `22.10.0`, you must explicitly enable DSL2 by adding `nextflow.enable.dsl=2` to the top of the script or by using the `-dsl2` command-line option. -::: - -This script defines two processes. The first splits a string into 6-character chunks, writing each one to a file with the prefix `chunk_`, and the second receives these files and transforms their contents to uppercase letters. The resulting strings are emitted on the `result` channel and the final output is printed by the `view` operator. - -Execute the script by entering the following command in your terminal: - -```console -$ nextflow run tutorial.nf - -N E X T F L O W ~ version 22.10.0 -executor > local (3) -[69/c8ea4a] process > splitLetters [100%] 1 of 1 ✔ -[84/c8b7f1] process > convertToUpper [100%] 2 of 2 ✔ -HELLO -WORLD! -``` - -You can see that the first process is executed once, and the second twice. Finally the result string is printed. - -It's worth noting that the process `convertToUpper` is executed in parallel, so there's no guarantee that the instance processing the first split (the chunk `Hello`) will be executed before the one processing the second split (the chunk `world!`). - -Thus, it is perfectly possible that you will get the final result printed out in a different order: - -``` -WORLD! -HELLO -``` - -:::{tip} -The hexadecimal string, e.g. `22/7548fa`, is the unique hash of a task, and the prefix of the directory where the task is executed. You can inspect a task's files by changing to the directory `$PWD/work` and using this string to find the specific task directory. -::: - -(getstarted-resume)= - -### Modify and resume - -Nextflow keeps track of all the processes executed in your pipeline. If you modify some parts of your script, only the processes that are actually changed will be re-executed. The execution of the processes that are not changed will be skipped and the cached result used instead. - -This helps a lot when testing or modifying part of your pipeline without having to re-execute it from scratch. - -For the sake of this tutorial, modify the `convertToUpper` process in the previous example, replacing the process script with the string `rev $x`, so that the process looks like this: - -```groovy -process convertToUpper { - input: - path x - output: - stdout - - """ - rev $x - """ -} -``` - -Then save the file with the same name, and execute it by adding the `-resume` option to the command line: - -```bash -nextflow run tutorial.nf -resume -``` - -It will print output similar to this: - -``` -N E X T F L O W ~ version 22.10.0 -executor > local (2) -[69/c8ea4a] process > splitLetters [100%] 1 of 1, cached: 1 ✔ -[d0/e94f07] process > convertToUpper [100%] 2 of 2 ✔ -olleH -!dlrow -``` - -You will see that the execution of the process `splitLetters` is actually skipped (the process ID is the same), and its results are retrieved from the cache. The second process is executed as expected, printing the reversed strings. - -:::{tip} -The pipeline results are cached by default in the directory `$PWD/work`. Depending on your script, this folder can take up a lot of disk space. It's a good idea to clean this folder periodically, as long as you know you won't need to resume any pipeline runs. -::: - -For more information, see the {ref}`cache-resume-page` page. - -(getstarted-params)= - -### Pipeline parameters - -Pipeline parameters are simply declared by prepending to a variable name the prefix `params`, separated by dot character. Their value can be specified on the command line by prefixing the parameter name with a double dash character, i.e. `--paramName` - -For the sake of this tutorial, you can try to execute the previous example specifying a different input string parameter, as shown below: - -```bash -nextflow run tutorial.nf --str 'Bonjour le monde' -``` - -The string specified on the command line will override the default value of the parameter. The output will look like this: - -``` -N E X T F L O W ~ version 22.10.0 -executor > local (4) -[8b/16e7d7] process > splitLetters [100%] 1 of 1 ✔ -[eb/729772] process > convertToUpper [100%] 3 of 3 ✔ -m el r -edno -uojnoB -``` - -:::{versionchanged} 20.11.0-edge -Any `.` (dot) character in a parameter name is interpreted as the delimiter of a nested scope. For example, `--foo.bar Hello` will be interpreted as `params.foo.bar`. If you want to have a parameter name that contains a `.` (dot) character, escape it using the back-slash character, e.g. `--foo\.bar Hello`. -::: diff --git a/docs/google.md b/docs/google.md index 39cabf5722..69ee86f83f 100644 --- a/docs/google.md +++ b/docs/google.md @@ -35,6 +35,9 @@ Then, define the following variable replacing the path in the example with the o export GOOGLE_APPLICATION_CREDENTIALS="/path/your/file/creds.json" ``` +See [Get started with Nextflow on Google Cloud Batch](https://www.nextflow.io/blog/2023/nextflow-with-gbatch.html) for more information on how to use Google Cloud Batch, +including how to set the required roles for your service account. + (google-batch)= ## Cloud Batch @@ -236,7 +239,7 @@ process.scratch = false tower.accessToken = '' ``` -The [Tower](https://cloud.tower.nf) access token is optional, but it enables higher API rate limits for the {ref}`wave-page` service required by Fusion. +The [Seqera Platform](https://seqera.io) access token is optional, but it enables higher API rate limits for the {ref}`wave-page` service required by Fusion. By default, Fusion mounts a local SSD disk to the VM at `/tmp`, using a machine type that can attach local SSD disks. If you specify your own machine type or machine series, they should be able to attach local SSD disks, otherwise the job scheduling will fail. diff --git a/docs/ignite.md b/docs/ignite.md deleted file mode 100644 index 4c3aa6bae9..0000000000 --- a/docs/ignite.md +++ /dev/null @@ -1,196 +0,0 @@ -(ignite-page)= - -# Apache Ignite - -:::{warning} -This feature is no longer maintained. -::: - -:::{versionchanged} 22.01.0-edge -The `ignite` executor must be enabled via the `nf-ignite` plugin. -::: - -Nextflow can be deployed in a *cluster* mode by using [Apache Ignite](https://ignite.apache.org/), an in-memory data-grid and clustering platform. - -Apache Ignite is packaged with Nextflow itself, so you won't need to install it separately or configure other third party software. - -(ignite-daemon)= - -## Cluster daemon - -In order to setup a cluster you will need to run a cluster daemon on each node within your cluster. If you want to support the {ref}`Docker integration ` provided by Nextflow, the Docker engine has to be installed and must run in each node. - -In its simplest form just launch the Nextflow daemon in each cluster node as shown below: - -```bash -nextflow node -bg -``` - -The command line option `-bg` launches the node daemon in the background. The output is stored in the log file `.node-nextflow.log`. The daemon process `PID` is saved in the file `.nextflow.pid` in the same folder. - -### Multicast discovery - -By default, the Ignite daemon tries to automatically discover all members in the cluster by using the network *multicast* discovery. Note that NOT all networks support this feature (for example Amazon AWS does not). - -:::{tip} -To check if multicast is available in your network, use the [iperf](http://sourceforge.net/projects/iperf/) tool. To test multicast, open a terminal on two machines within the network and run the following command in the first one `iperf -s -u -B 228.1.2.4 -i 1` and `iperf -c 228.1.2.4 -u -T 32 -t 3 -i 1` on the second. If data is being transferred then multicast is working. -::: - -Ignite uses the multicast group `228.1.2.4` and port `47400` by default. You can change these values, by using the `cluster.join` command line option, as shown below: - -```bash -nextflow node -cluster.join multicast:224.2.2.3:55701 -``` - -In case multicast discovery is not available in your network, you can try one of the following alternative methods: - -### Shared file system - -Simply provide a path shared across the cluster by a network file system, as shown below: - -```bash -nextflow node -bg -cluster.join path:/net/shared/cluster -``` - -The cluster members will use that path to discover each other. - -### IP addresses - -Provide a list of pre-configured IP addresses on the daemon launch command line, for example: - -```bash -nextflow node -cluster.join ip:10.0.2.1,10.0.2.2,10.0.2.4 -``` - -### Advanced options - -The following cluster node configuration options can be used. - -| Name | Description | -| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| join | IP address(es) of one or more cluster nodes to which the daemon will join. | -| group | The name of the cluster to which this node join. It allows the creation of separate cluster instances. Default: `nextflow` | -| maxCpus | Max number of CPUs that can be used by the daemon to run user tasks. By default it is equal to the number of CPU cores. | -| maxDisk | Max amount of disk storage that can be used by user tasks e.g. `500 GB`. | -| maxMemory | Max amount of memory that can be used by user tasks e.g. `64 GB`. Default total available memory. | -| interface | Network interfaces that Ignite will use. It can be the interface IP address or name. | -| failureDetectionTimeout | Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed. | -| clientFailureDetectionTimeout | Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed. | -| tcp.localAddress | Defines the local host IP address. | -| tcp.localPort | Defines the local port to listen to. | -| tcp.localPortRange | Range for local ports. | -| tcp.reconnectCount | Number of times the node tries to (re)establish connection to another node. | -| tcp.networkTimeout | Defines the network timeout. | -| tcp.socketTimeout | Defines the socket operations timeout. This timeout is used to limit connection time and write-to-socket time. Note that when running Ignite on Amazon EC2, socket timeout must be set to a value significantly greater than the default (e.g. to 30000). | -| tcp.ackTimeout | Defines the timeout for receiving acknowledgement for sent message. | -| tcp.maxAckTimeout | Defines the maximum timeout for receiving acknowledgement for sent message. | -| tcp.joinTimeout | Defines the join timeout. | - -These options can be specified as command line parameters by adding the prefix `-cluster.` to them, as shown below: - -```bash -nextflow node -bg -cluster.maxCpus 4 -cluster.interface eth0 -``` - -The same options can be entered into the Nextflow {ref}`configuration file`, as shown below: - -```groovy -cluster { - join = 'ip:192.168.1.104' - interface = 'eth0' -} -``` - -Finally daemon options can be provided also as environment variables having the name in upper-case and by adding the prefix `NXF_CLUSTER_` to them, for example: - -```bash -export NXF_CLUSTER_JOIN='ip:192.168.1.104' -export NXF_CLUSTER_INTERFACE='eth0' -``` - -## Pipeline execution - -The pipeline execution needs to be launched in a `head` node i.e. a cluster node where the Nextflow node daemon is **not** running. In order to execute your pipeline in the Ignite cluster you will need to use the Ignite executor, as shown below: - -```bash -nextflow run -process.executor ignite -``` - -If your network does no support multicast discovery, you will need to specify the `joining` strategy as you did for the cluster daemons. For example, using a shared path: - -```bash -nextflow run -process.executor ignite -cluster.join path:/net/shared/path -``` - -## Execution with MPI - -Nextflow is able to deploy and self-configure an Ignite cluster on demand, taking advantage of the Open [MPI](https://en.wikipedia.org/wiki/Message_Passing_Interface) standard that is commonly available in grid and supercomputer facilities. - -In this scenario a Nextflow workflow needs to be executed as an MPI job. Under the hood Nextflow will launch a *driver* process in the first of the nodes, allocated by your job request, and an Ignite daemon in the remaining nodes. - -In practice you will need a launcher script to submit an MPI job request to your batch scheduler/resource manager. The batch scheduler must reserve the compute nodes in an exclusive manner to avoid having multiple Ignite daemons running on the same node. Nextflow must be launched using the `mpirun` utility, as if it were an MPI application, specifying the `--pernode` option. - -### Platform LSF launcher - -The following example shows a launcher script for the [Platform LSF](https://en.wikipedia.org/wiki/Platform_LSF/) resource manager: - -```bash -#!/bin/bash -#BSUB -oo output_%J.out -#BSUB -eo output_%J.err -#BSUB -J -#BSUB -q -#BSUB -W 02:00 -#BSUB -x -#BSUB -n 80 -#BSUB -M 10240 -#BSUB -R "span[ptile=16]" -export NXF_CLUSTER_SEED=$(shuf -i 0-16777216 -n 1) -mpirun --pernode nextflow run -with-mpi [pipeline parameters] -``` - -It requests 5 nodes (80 processes, with 16 cpus per node). The `-x` directive allocates the node in an exclusive manner. Nextflow needs to be executed using the `-with-mpi` command line option. It will automatically use `ignite` as the executor. - -The variable `NXF_CLUSTER_SEED` must contain an integer value (in the range 0-16777216) that will unequivocally identify your cluster instance. In the above example it is randomly generated by using the [shuf](http://linux.die.net/man/1/shuf) Linux tool. - -### Univa Grid Engine launcher - -The following example shows a launcher script for the [Univa Grid Engine](https://en.wikipedia.org/wiki/Univa_Grid_Engine) (aka SGE): - -```bash -#!/bin/bash -#$ -cwd -#$ -j y -#$ -o -#$ -l virtual_free=10G -#$ -q -#$ -N -#$ -pe ompi 5 -export NXF_CLUSTER_SEED=$(shuf -i 0-16777216 -n 1) -mpirun --pernode nextflow run -with-mpi [pipeline parameters] -``` - -As in the previous script it allocates 5 processing nodes. UGE/SGE does not have an option to reserve a node in an exclusive manner. A common workaround is to request the maximum amount of memory or cpus available in the nodes of your cluster. - -### Linux SLURM launcher - -When using Linux SLURM you will need to use `srun` instead `mpirun` in your launcher script. For example: - -```bash -#!/bin/bash -#SBATCH --job-name= -#SBATCH --output= -#SBATCH --ntasks=5 -#SBATCH --cpus-per-task=16 -#SBATCH --tasks-per-node=1 -export NXF_CLUSTER_SEED=$(shuf -i 0-16777216 -n 1) -srun nextflow run hello.nf -with-mpi -``` - -As before, this allocates 5 processing nodes (`--ntasks=5`) and each node will be able to use up to 16 cpus (`--cpus-per-task=16`). When using SLURM it's not necessary to allocate compute nodes in an exclusive manner. It's even possible to launch more than one Nextflow daemon instance per node, though not suggested. - -To submit the pipeline execution create a file like the above, then use the following command: - -```bash -sbatch -``` diff --git a/docs/index.md b/docs/index.md index a8a113ad90..69476d91c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,8 @@ -# Nextflow's documentation! +# Nextflow + +*"Dataflow variables are spectacularly expressive in concurrent programming"* +
[Henri E. Bal , Jennifer G. Steiner , Andrew S. Tanenbaum](https://dl.acm.org/doi/abs/10.1145/72551.72552) [![Nextflow CI](https://github.com/nextflow-io/nextflow/workflows/Nextflow%20CI/badge.svg)](https://github.com/nextflow-io/nextflow/actions/workflows/build.yml?query=branch%3Amaster+event%3Apush) [![Nextflow version](https://img.shields.io/github/release/nextflow-io/nextflow.svg?colorB=58bd9f&style=popout)](https://github.com/nextflow-io/nextflow/releases/latest) @@ -8,25 +11,44 @@ [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg?colorB=58bd9f&style=popout)](http://bioconda.github.io/recipes/nextflow/README.html) [![Nextflow license](https://img.shields.io/github/license/nextflow-io/nextflow.svg?colorB=58bd9f&style=popout)](https://github.com/nextflow-io/nextflow/blob/master/COPYING) -Nextflow is a workflow system for creating scalable, portable, and reproducible workflows. +Nextflow is a workflow system for creating scalable, portable, and reproducible workflows. It is based on the dataflow programming model, which greatly simplifies the writing of parallel and distributed pipelines, allowing you to focus on the flow of data and computation. Nextflow can deploy workflows on a variety of execution platforms, including your local machine, HPC schedulers, AWS Batch, Azure Batch, Google Cloud Batch, and Kubernetes. Additionally, it supports many ways to manage your software dependencies, including Conda, Spack, Docker, Podman, Singularity, and more. + +## Get started + +- Get an {ref}`overview ` of Nextflow and its key concepts. +- Get started with Nextflow by {ref}`installing ` it and running {ref}`your first script `. +- Check out [this blog post](https://www.nextflow.io/blog/2023/learn-nextflow-in-2023.html) for even more resources on how to learn Nextflow. + +## Community + +You can post questions and get help in the [Nextflow community forum](https://community.seqera.io) or the [Nextflow Slack](https://www.nextflow.io/slack-invite.html). Bugs and feature requests should be reported as [GitHub issues](https://github.com/nextflow-io/nextflow/issues/new/choose). + +The Nextflow community is highly active with regular community meetings, events, a podcast and more. You can view much of this material on the [Nextflow](https://www.youtube.com/@Nextflow) and [nf-core](https://www.youtube.com/@nf-core) YouTube channels. + +The [nf-core](https://nf-co.re/) project is a community effort aggregating high quality Nextflow workflows which can be used by everyone. + +## Contributing + +Contributions are more than welcome. See the {ref}`Contributing ` page for details. -## Rationale +## License -The rise of big data has made it increasingly necessary to be able to analyze and perform experiments on large datasets in a portable and reproducible manner. +Nextflow is released under the Apache 2.0 license. Nextflow is a [registered trademark](https://github.com/nextflow-io/trademark). -Parallelization and distributed computing are the best ways to tackle this challenge, but the tools commonly available to computational scientists often lack good support for these techniques, or they provide a model that fits poorly with the needs of computational scientists and often require knowledge of complex tools and APIs. +## Citations -The Nextflow language is inspired by [the Unix philosophy](https://en.wikipedia.org/wiki/Unix_philosophy), in which many simple command line tools can be chained together into increasingly complex tasks. Similarly, a Nextflow script consists of composing many simple processes into increasingly complex pipelines. Each process executes a given tool or scripting language, and by specifying the process inputs and outputs, Nextflow coordinates the execution of tasks for you. +If you use Nextflow in your work, please cite: -The Nextflow runtime integrates with many popular execution platforms (HPC schedulers, cloud providers) and software tools (Git, Docker, Conda), allowing you to fully describe a computational pipeline with all of its dependencies and run it in nearly any environment -- write once, run anywhere. +P. Di Tommaso, et al. Nextflow enables reproducible computational workflows. Nature Biotechnology 35, 316–319 (2017) doi:[10.1038/nbt.3820](http://www.nature.com/nbt/journal/v35/n4/full/nbt.3820.html) ```{toctree} :hidden: -:caption: Introduction +:caption: Get started :maxdepth: 1 -getstarted -basic +overview +install +your-first-script ``` ```{toctree} @@ -91,7 +113,6 @@ kubernetes :maxdepth: 1 flux -ignite ``` ```{toctree} @@ -100,6 +121,7 @@ ignite :maxdepth: 1 developer/index +developer/diagram developer/packages developer/plugins ``` diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000000..1b0de63463 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,118 @@ +(install-page)= + +# Installation + +(install-requirements)= + +## Requirements + +Nextflow can be used on any POSIX-compatible system (Linux, macOS, etc), and on Windows through [WSL](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux). It requires Bash 3.2 (or later) and [Java 11 (or later, up to 21)](http://www.oracle.com/technetwork/java/javase/downloads/index.html) to be installed. You can see which version you have using the following command: + +```bash +java -version +``` + +If you don't have a compatible version of Java installed in your computer, it is recommended that you install it through [SDKMAN!](https://sdkman.io/), and that you use the latest LTS version of Temurin. See [this website](https://whichjdk.com/) for more information. + +To install Java with SDKMAN: + +1. Install SDKMAN: + + ```bash + curl -s https://get.sdkman.io | bash + ``` + +2. Open a new terminal and install Java: + + ```bash + sdk install java 17.0.10-tem + ``` + +3. Confirm that Java is installed correctly: + + ```bash + java -version + ``` + +(install-nextflow)= + +## Install Nextflow + +Nextflow is distributed as a self-installing package, in order to make the installation process as simple as possible: + +1. Install Nextflow: + + ```bash + curl -s https://get.nextflow.io | bash + ``` + + This will create the `nextflow` executable in the current directory. + + :::{tip} + You can set `export CAPSULE_LOG=none` to make the installation logs less verbose. + ::: + +2. Make Nextflow executable: + + ```bash + chmod +x nextflow + ``` + +3. Move Nextflow into an executable path: + + ```bash + sudo mv nextflow /usr/local/bin + ``` + +4. Confirm that Nextflow is installed correctly: + + ```bash + nextflow info + ``` + +## Updates + +With Nextflow installed in your environment, you can update to the latest version using the following command: + +```bash +nextflow self-update +``` + +You can also temporarily switch to a specific version of Nextflow with the `NXF_VER` environment variable. For example: + +```bash +NXF_VER=23.10.0 nextflow run hello +``` + +## Stable and edge releases + +A *stable* version of Nextflow is released every six months, in the 4th and 10th month of each year. + +Additionally, an *edge* version is released on a monthly basis. The edge releases can be used to access the latest updates and experimental features. + +To use the latest edge release, set `NXF_EDGE=1` when updating: + +```bash +NXF_EDGE=1 nextflow self-update +``` + +You can also use `NXF_VER` to switch to any edge release: + +```bash +$ nextflow info +``` + +## Standalone distribution + +Nextflow has a set of {ref}`core plugins ` which are downloaded at runtime by default. There is also a standalone distribution (i.e. the `all` distribution) which comes pre-packaged with all core plugins. This distribution is mainly useful for offline environments. + +The installer for the `all` distribution can be found on the [GitHub releases page](https://github.com/nextflow-io/nextflow/releases), under the "Assets" section for a specific release. The installation procedure is the same as for the standard distribution, only using this URL instead of `https://get.nextflow.io`: + +```bash +export NXF_VER=23.10.0 +curl -s https://github.com/nextflow-io/nextflow/releases/download/v$NXF_VER/nextflow-$NXF_VER-all +``` + +:::{warning} +The `all` distribution does not support third-party plugins. Only the {ref}`core plugins ` are supported. +::: diff --git a/docs/metadata.md b/docs/metadata.md index 529060bb98..4fdc3faeb2 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -58,6 +58,12 @@ The following table lists the properties that can be accessed on the `workflow` : *Available only in the `workflow.onComplete` and `workflow.onError` handlers* : Exit status of the task that caused the workflow execution to fail. +`workflow.fusion.enabled` +: Whether Fusion is enabled. + +`workflow.fusion.version` +: Fusion version in use. + `workflow.homeDir` : User system home directory. @@ -67,6 +73,11 @@ The following table lists the properties that can be accessed on the `workflow` `workflow.manifest` : Entries of the workflow manifest. +`workflow.preview` +: :::{versionadded} 24.04.0 + ::: +: Returns `true` whenever the current instance is a preview execution. + `workflow.profile` : Used configuration profile. @@ -111,6 +122,9 @@ The following table lists the properties that can be accessed on the `workflow` `workflow.userName` : User system account name. +`workflow.wave.enabled` +: Whether Wave is enabled. + `workflow.workDir` : Workflow working directory. diff --git a/docs/operator.md b/docs/operator.md index 8ba07a7cc7..6eab124a35 100644 --- a/docs/operator.md +++ b/docs/operator.md @@ -21,7 +21,7 @@ This page is a comprehensive reference for all Nextflow operators. However, if y :::{versionadded} 19.08.0-edge ::: -*Returns: map of queue channels* +*Returns: multiple queue channels or value channels, matching the source type* The `branch` operator forwards each item from a source channel to one of multiple output channels, based on a selection criteria. @@ -105,7 +105,7 @@ This operator has multiple variants: :language: console ``` -`buffer( size: n )` +`buffer( size: n, remainder: true | false )` : Emits a new subset for every `n` items. Remaining items are discarded. For example: @@ -127,7 +127,7 @@ This operator has multiple variants: :language: console ``` -`buffer( size: n, skip: m )` +`buffer( size: n, skip: m, remainder: true | false )` : Emits a new subset for every `n` items, skipping `m` items before collecting each subset. For example: @@ -486,7 +486,7 @@ See also: [combine](#combine) ## distinct -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `distinct` operator forwards a source channel with *consecutively* repeated items removed, such that each emitted item is different from the preceding one: @@ -514,7 +514,7 @@ See also: [unique](#unique) ## dump -*Returns: queue channel or value channel, depending on the input* +*Returns: queue channel or value channel, matching the source type* The `dump` operator prints each item in a source channel when the pipeline is executed with the `-dump-channels` command-line option, otherwise it does nothing. It is a useful way to inspect and debug channels quickly without having to modify the pipeline script. @@ -538,7 +538,7 @@ Available options: ## filter -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `filter` operator emits the items from a source channel that satisfy a condition, discarding all other items. The filter condition can be a literal value, a {ref}`regular expression `, a type qualifier (i.e. Java class), or a boolean predicate. @@ -696,7 +696,7 @@ Available options: ## ifEmpty -*Returns: value channel* +*Returns: queue channel or value channel, matching the source type* The `ifEmpty` operator emits a source channel, or a default value if the source channel is *empty* (doesn't emit any value): @@ -788,7 +788,7 @@ The `last` operator emits the last item from a source channel: ## map -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `map` operator applies a *mapping function* to each item from a source channel: @@ -840,7 +840,7 @@ The following examples show how to find the longest string in a channel: ## merge -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `merge` operator joins the items from two or more channels into a new channel: @@ -862,6 +862,12 @@ An optional closure can be used to control how two items are merged: :language: console ``` +The `merge` operator may return a queue channel or value channel depending on the inputs: + +- If the first argument is a queue channel, the `merge` operator will return a queue channel merging as many values as are available for all inputs. Value channels will be re-used for each merged value. + +- If the first argument is a value channel, the `merge` operator will return a value channel merging the first value from each input, regardless of whether there are queue channel inputs with additional values. + :::{danger} In general, the use of the `merge` operator is discouraged. Processes and channel operators are not guaranteed to emit items in the order that they were received, as they are executed concurrently. Therefore, if you try to merge output channels from different processes, the resulting channel may be different on each run, which will cause resumed runs to {ref}`not work properly `. @@ -940,7 +946,7 @@ See also: [concat](#concat) :::{versionadded} 19.11.0-edge ::: -*Returns: map of queue channels* +*Returns: multiple queue channels or value channels, matching the source type* The `multiMap` operator applies a set of mapping functions to a source channel, producing a separate output channel for each mapping function. @@ -1397,7 +1403,7 @@ See also: [countLines](#countlines) ## subscribe -*Returns: nothing* +*Returns: the source channel* The `subscribe` operator invokes a custom function for each item from a source channel: @@ -1466,6 +1472,8 @@ An optional {ref}`closure ` can be used to transform each item b :language: console ``` +(operator-take)= + ## take *Returns: queue channel* @@ -1488,7 +1496,7 @@ See also: [until](#until) ## tap -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `tap` operator assigns a source channel to a variable, and emits the source channel. It is a useful way to extract intermediate output channels from a chain of operators. For example: @@ -1504,7 +1512,7 @@ See also: [set](#set) ## toInteger -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `toInteger` operator converts string values from a source channel to integer values: @@ -1641,7 +1649,7 @@ See also: [groupTuple](#grouptuple) ## unique -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `unique` operator emits the unique items from a source channel: @@ -1669,9 +1677,11 @@ The difference between `unique` and `distinct` is that `unique` removes *all* du See also: [distinct](#distinct) +(operator-until)= + ## until -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `until` operator emits each item from a source channel until a stopping condition is satisfied: @@ -1689,7 +1699,7 @@ See also: [take](#take) ## view -*Returns: queue channel* +*Returns: queue channel or value channel, matching the source type* The `view` operator prints each item from a source channel to standard output: diff --git a/docs/basic.md b/docs/overview.md similarity index 65% rename from docs/basic.md rename to docs/overview.md index 6f7b64b74e..f848facde4 100644 --- a/docs/basic.md +++ b/docs/overview.md @@ -1,10 +1,14 @@ -# Basic concepts +(overview-page)= -Nextflow is a reactive workflow framework and a programming [DSL](http://en.wikipedia.org/wiki/Domain-specific_language) that eases the writing of data-intensive computational pipelines. +# Overview -It is designed around the idea that the Linux platform is the lingua franca of data science. Linux provides many simple but powerful command-line and scripting tools that, when chained together, facilitate complex data manipulations. +## Why Nextflow? -Nextflow extends this approach, adding the ability to define complex program interactions and a high-level parallel computational environment based on the *dataflow* programming model. +The rise of big data has made it increasingly necessary to be able to analyze and perform experiments on large datasets in a portable and reproducible manner. Parallelization and distributed computing are the best ways to tackle this challenge, but the tools commonly available to computational scientists often lack good support for these techniques, or they provide a model that fits poorly with the needs of computational scientists and often require knowledge of complex tools and APIs. Nextflow was created to address these challenges. + +The Nextflow language is inspired by [the Unix philosophy](https://en.wikipedia.org/wiki/Unix_philosophy), in which many simple command line tools can be chained together into increasingly complex tasks. Similarly, a Nextflow script consists of composing many simple processes into increasingly complex pipelines. Each process executes a given tool or scripting language, and by specifying the process inputs and outputs, Nextflow coordinates the execution of tasks for you. + +The Nextflow runtime integrates with many popular execution platforms (HPC schedulers, cloud providers) and software tools (Git, Docker, Conda), allowing you to fully describe a computational pipeline with all of its dependencies and run it in nearly any environment -- write once, run anywhere. ## Processes and channels @@ -17,45 +21,45 @@ Any process can define one or more channels as *input* and *output*. The interac A Nextflow script looks like this: ```groovy -// Declare syntax version -nextflow.enable.dsl=2 - // Script parameters params.query = "/some/data/sample.fa" params.db = "/some/path/pdb" process blastSearch { input: - path query - path db + path query + path db + output: - path "top_hits.txt" + path "top_hits.txt" - """ - blastp -db $db -query $query -outfmt 6 > blast_result - cat blast_result | head -n 10 | cut -f 2 > top_hits.txt - """ + """ + blastp -db $db -query $query -outfmt 6 > blast_result + cat blast_result | head -n 10 | cut -f 2 > top_hits.txt + """ } process extractTopHits { input: - path top_hits + path top_hits + path db output: - path "sequences.txt" + path "sequences.txt" - """ - blastdbcmd -db $db -entry_batch $top_hits > sequences.txt - """ + """ + blastdbcmd -db $db -entry_batch $top_hits > sequences.txt + """ } workflow { def query_ch = Channel.fromPath(params.query) - blastSearch(query_ch, params.db) | extractTopHits | view + blastSearch(query_ch, params.db) + extractTopHits(blastSearch.out, params.db).view() } ``` -The above example defines two processes. Their execution order is not determined by the fact that the `blastSearch` process comes before `extractTopHits` in the script (it could also be written the other way around). Instead, the pipe operator (`|`) in the workflow between `blastSearch` and `extractTopHits` forwards the outputs from one process to the inputs of the following one. +The above example defines two processes. Their execution order is not determined by the fact that the `blastSearch` process comes before `extractTopHits` in the script (it could also be written the other way around). Instead, execution order is determined by their _dependencies_ -- `extractTopHits` depends on the output of `blastSearch`, so `blastSearch` will be executed first, and then `extractTopHits`. When the workflow is started, it will create two processes and one channel (`query_ch`) and it will link all of them. Both processes will be started at the same time and they will listen to their respective input channels. Whenever `blastSearch` emits a value, `extractTopHits` will receive it (i.e. `extractTopHits` consumes the channel in a *reactive* way). @@ -71,18 +75,19 @@ In other words, Nextflow provides an abstraction between the pipeline's function The following batch schedulers are supported: -- [Open grid engine](http://gridscheduler.sourceforge.net/) -- [Univa grid engine](http://www.univa.com/) +- [Open Grid Engine](http://gridscheduler.sourceforge.net/) +- [Univa Grid Engine](http://www.univa.com/) - [Platform LSF](http://www.ibm.com/systems/technicalcomputing/platformcomputing/products/lsf/) -- [Linux SLURM](https://computing.llnl.gov/linux/slurm/) +- [SLURM](https://computing.llnl.gov/linux/slurm/) - [Flux Framework](https://flux-framework.org/) -- [PBS Works](http://www.pbsworks.com/gridengine/) +- [PBS](http://www.pbsworks.com/gridengine/) - [Torque](http://www.adaptivecomputing.com/products/open-source/torque/) - [HTCondor](https://research.cs.wisc.edu/htcondor/) The following cloud platforms are supported: - [Amazon Web Services (AWS)](https://aws.amazon.com/) +- [Microsoft Azure](https://azure.microsoft.com/) - [Google Cloud Platform (GCP)](https://cloud.google.com/) - [Kubernetes](https://kubernetes.io/) @@ -96,10 +101,6 @@ Nextflow scripting is an extension of the [Groovy programming language]( - - - ## Configuration options Pipeline configuration properties are defined in a file named `nextflow.config` in the pipeline execution directory. diff --git a/docs/plugins.md b/docs/plugins.md index 1ceeb18bb1..e3213cc77f 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -4,17 +4,19 @@ Nextflow has a plugin system that allows the use of extensible components that are downloaded and installed at runtime. +(plugins-core)= + ## Core plugins The following functionalities are provided via plugin components, and they make part of the Nextflow *core* plugins: -- `nf-amazon`: Support for Amazon cloud. -- `nf-azure`: Support for Azure cloud. +- `nf-amazon`: Support for Amazon Web Services. +- `nf-azure`: Support for Microsoft Azure. - `nf-cloudcache`: Support for the cloud cache (see `NXF_CLOUDCACHE_PATH` under {ref}`config-env-vars`). - `nf-console`: Implement Nextflow [REPL console](https://www.nextflow.io/blog/2015/introducing-nextflow-console.html). - `nf-ga4gh`: Support [GA4GH APIs](https://www.ga4gh.org/). -- `nf-google`: Support for Google cloud. -- `nf-tower`: Support for [Tower](https://tower.nf) cloud platform. +- `nf-google`: Support for Google Cloud. +- `nf-tower`: Support for [Seqera Platform](https://seqera.io) (formerly Tower Cloud). - `nf-wave`: Support for [Wave containers](https://seqera.io/wave/) service. @@ -173,7 +175,7 @@ import nextflow.Session import nextflow.plugin.extension.Function import nextflow.plugin.extension.PluginExtensionPoint -class MyExtension implements PluginExtensionPoint { +class MyExtension extends PluginExtensionPoint { @Override void init(Session session) {} @@ -217,7 +219,7 @@ import nextflow.plugin.extension.Factory import nextflow.plugin.extension.Operator import nextflow.plugin.extension.PluginExtensionPoint -class MyExtension implements PluginExtensionPoint { +class MyExtension extends PluginExtensionPoint { @Override void init(Session session) {} @@ -377,7 +379,7 @@ nextflow run -plugins nf-hello To use Nextflow plugins in an offline environment: -1. {ref}`Download Nextflow ` and install it on a system with an internet connection. Do not use the "all" package, as this does not allow the use of custom plugins. +1. {ref}`Download Nextflow ` and install it on a system with an internet connection. Do not use the "all" package, as this does not allow the use of custom plugins. 2. Download any additional plugins by running `nextflow plugin install `. Alternatively, simply run your pipeline once and Nextflow will download all of the plugins that it needs. diff --git a/docs/process.md b/docs/process.md index 68e069a872..1a65a44168 100644 --- a/docs/process.md +++ b/docs/process.md @@ -1008,7 +1008,7 @@ Some caveats on glob pattern behavior: Although the input files matching a glob output declaration are not included in the resulting output channel, these files may still be transferred from the task scratch directory to the original task work directory. Therefore, to avoid unnecessary file copies, avoid using loose wildcards when defining output files, e.g. `path '*'`. Instead, use a prefix or a suffix to restrict the set of matching files to only the expected ones, e.g. `path 'prefix_*.sorted.bam'`. ::: -Read more about glob syntax at the following link [What is a glob?][what is a glob?] +Read more about glob syntax at the following link [What is a glob?][glob] ### Dynamic output file names @@ -1049,43 +1049,41 @@ To sum up, the use of output files with static names over dynamic ones is prefer The `env` qualifier allows you to output a variable defined in the process execution environment: -```groovy -process myTask { - output: - env FOO - - script: - ''' - FOO=$(ls -la) - ''' -} - -workflow { - myTask | view { "directory contents: $it" } -} +```{literalinclude} snippets/process-out-env.nf +:language: groovy ``` +:::{versionchanged} 23.12.0-edge +Prior to this version, if the environment variable contained multiple lines of output, the output would be compressed to a single line by converting newlines to spaces. +::: + (process-stdout)= ### Output type `stdout` The `stdout` qualifier allows you to output the `stdout` of the executed process: -```groovy -process sayHello { - output: - stdout +```{literalinclude} snippets/process-stdout.nf +:language: groovy +``` - """ - echo Hello world! - """ -} +(process-out-eval)= -workflow { - sayHello | view { "I say... $it" } -} +### Output type `eval` + +:::{versionadded} 24.02.0-edge +::: + +The `eval` qualifier allows you to capture the standard output of an arbitrary command evaluated the task shell interpreter context: + +```{literalinclude} snippets/process-out-eval.nf +:language: groovy ``` +Only one-line Bash commands are supported. You can use a semi-colon `;` to specify multiple Bash commands on a single line, and many interpreters can execute arbitrary code on the command line, e.g. `python -c 'print("Hello world!")'`. + +If the command fails, the task will also fail. In Bash, you can append `|| true` to a command to suppress any command failure. + (process-out-tuple)= ### Output type `tuple` @@ -1333,6 +1331,65 @@ Allowed values for the `arch` directive are as follows, grouped by equivalent fa Examples of values for the architecture `target` option are `cascadelake`, `icelake`, `zen2` and `zen3`. See the Spack documentation for the full and up-to-date [list of meaningful targets](https://spack.readthedocs.io/en/latest/basic_usage.html#support-for-specific-microarchitectures). +(process-array)= + +### array + +:::{versionadded} 24.04.0 +::: + +:::{warning} *Experimental: may change in a future release.* +::: + +The `array` directive allows you to submit tasks as *job arrays* for executors that support it. + +A job array is a collection of jobs with the same resource requirements and the same script (parameterized by an index). Job arrays incur significantly less scheduling overhead compared to individual jobs, and as a result they are preferred by HPC schedulers where possible. + +The directive should be specified with a given array size, along with an executor that supports job arrays. For example: + +```groovy +process cpu_task { + executor 'slurm' + array 100 + + ''' + your_command --here + ''' +} +``` + +Nextflow currently supports job arrays for the following executors: + +- {ref}`awsbatch-executor` +- {ref}`google-batch-executor` +- {ref}`lsf-executor` +- {ref}`pbs-executor` +- {ref}`pbspro-executor` +- {ref}`sge-executor` +- {ref}`slurm-executor` + +A process using job arrays will collect tasks and submit each batch as a job array when it is ready. Any "leftover" tasks will be submitted as a partial job array. + +Once a job array is submitted, each "child" task is executed as an independent job. Any tasks that fail (and can be retried) will be retried without interfering with the tasks that succeeded. Retried tasks are submitted individually rather than through a job array, in order to allow for the use of [dynamic resources](#dynamic-computing-resources). + +The following directives must be uniform across all tasks in a process that uses job arrays, because these directives are specified once for the entire job array: + +- {ref}`process-accelerator` +- {ref}`process-clusterOptions` +- {ref}`process-cpus` +- {ref}`process-disk` +- {ref}`process-machineType` +- {ref}`process-memory` +- {ref}`process-queue` +- {ref}`process-resourcelabels` +- {ref}`process-resourcelimits` +- {ref}`process-time` + +For cloud-based executors like AWS Batch, or when using Fusion with any executor, the following additional directives must be uniform: + +- {ref}`process-container` +- {ref}`process-containerOptions` + (process-beforescript)= ### beforeScript @@ -1393,6 +1450,30 @@ The following options are available: The `clusterOptions` directive allows the usage of any native configuration option accepted by your cluster submit command. You can use it to request non-standard resources or use settings that are specific to your cluster and not supported out of the box by Nextflow. +The cluster options can be a string: + +```groovy +process foo { + clusterOptions '-x 1 -y 2' + // ... +} +``` + +:::{versionchanged} 24.04.0 +Prior to this version, grid executors that require each option to be on a separate line in the job script would attempt to split multiple options using a variety of different conventions. Multiple options can now be specified more clearly using a string list as shown below. +::: + +The cluster options can also be a string list: + +```groovy +process foo { + clusterOptions '-x 1', '-y 2', '--flag' + // ... +} +``` + +Grid executors that require one option per line will write each option to a separate line, while grid executors that allow multiple options per line will write all options to a single line, the same as with a string. This form is useful to control how the options are split across lines when it is required by the scheduler. + :::{note} This directive is only used by grid executors. Refer to the {ref}`executor-page` page to see which executors support this directive. ::: @@ -1636,7 +1717,6 @@ The following executors are available: | `azurebatch` | [Azure Batch](https://azure.microsoft.com/en-us/services/batch/) service | | `condor` | [HTCondor](https://research.cs.wisc.edu/htcondor/) job scheduler | | `google-lifesciences` | [Google Genomics Pipelines](https://cloud.google.com/life-sciences) service | -| `ignite` | [Apache Ignite](https://ignite.apache.org/) cluster | | `k8s` | [Kubernetes](https://kubernetes.io/) cluster | | `local` | The computer where `Nextflow` is launched | | `lsf` | [Platform LSF](http://en.wikipedia.org/wiki/Platform_LSF) job scheduler | @@ -2151,6 +2231,11 @@ The following options are available: effect: "NoSchedule" ``` +`ttlSecondsAfterFinished` +: :::{versionadded} 24.02.0-edge + ::: +: Specifies the [TTL mechanism](https://kubernetes.io/docs/concepts/workloads/controllers/job/#ttl-mechanism-for-finished-jobs) for finished jobs in seconds. Applies to both successful and failed jobs. + `volumeClaim: '', mountPath: '' [, subPath: '', readOnly: true | false]` : *Can be specified multiple times* : Mounts a [Persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) with the given name to the given path. @@ -2217,7 +2302,10 @@ Available options: : Enable or disable the publish rule depending on the boolean value specified (default: `true`). `failOnError` -: When `true` abort the execution if some file can't be published to the specified target directory or bucket for any cause (default: `false`) +: :::{versionchanged} 24.03.0-edge + The default value was change from `false` to `true` + ::: +: When `true` abort the execution if some file can't be published to the specified target directory or bucket for any cause (default: `true`) `mode` : The file publishing method. Can be one of the following values: @@ -2327,6 +2415,43 @@ Resource labels in Azure are added to pools, rather than jobs, in order to facil See also: [label](#label) +(process-resourcelimits)= + +### resourceLimits + +:::{versionadded} 24.04.0 +::: + +The `resourceLimits` directive allows you to specify environment-specific limits for task resource requests. Resource limits can be specified in a process as follows: + +```groovy +process my_task { + resourceLimits cpus: 24, memory: 768.GB, time: 72.h + + script: + ''' + your_command --here + ''' +} +``` + +Or in the Nextflow configuration: + +```groovy +process { + resourceLimits = [ cpus: 24, memory: 768.GB, time: 72.h ] +} +``` + +Resource limits can be defined for the following directives: + +- [cpus](#cpus) +- [disk](#disk) +- [memory](#memory) +- [time](#time) + +Resource limits are a useful way to specify environment-specific limits alongside tasks with [dynamic resources](#dynamic-computing-resources). Normally, if a task requests more resources than can be provisioned (e.g. a task requests 32 cores but the largest node in the cluster has 24), the task will either fail or cause the pipeline to hang forever as it will never be scheduled. If the `resourceLimits` directive is defined with these limits, the task resources will be automatically reduced to comply with these limits before the job is submitted. + (process-scratch)= ### scratch @@ -2669,4 +2794,3 @@ process foo { ``` [glob]: http://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob -[what is a glob?]: http://docs.oracle.com/javase/tutorial/essential/io/fileOps.html#glob diff --git a/docs/script.md b/docs/script.md index f8471f76d4..3b7ea19642 100644 --- a/docs/script.md +++ b/docs/script.md @@ -533,6 +533,11 @@ The following variables are implicitly defined in the script global execution sc ::: : The directory where the main script is located. +`secrets` +: :::{versionadded} 24.02.0-edge + ::: +: Dictionary like object holding workflow secrets. Read the {ref}`secrets-page` page for more information. + `workDir` : The directory where tasks temporary files are created. @@ -1302,6 +1307,9 @@ The following methods are defined for Paths for splitting and counting records: `countLines()` : Counts the number of lines in a text file. See the {ref}`operator-splittext` operator for available options. +`splitCsv()` +: Splits a CSV file into a list of records. See the {ref}`operator-splitcsv` operator for available options. + `splitFasta()` : Splits a [FASTA](https://en.wikipedia.org/wiki/FASTA_format) file into a list of records. See the {ref}`operator-splitfasta` operator for available options. @@ -1311,5 +1319,5 @@ The following methods are defined for Paths for splitting and counting records: `splitJson()` : Splits a JSON file into a list of records. See the {ref}`operator-splitjson` operator for available options. -`splitLines()` +`splitText()` : Splits a text file into a list of lines. See the {ref}`operator-splittext` operator for available options. diff --git a/docs/secrets.md b/docs/secrets.md index 1f6253a69f..e75368271f 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -47,7 +47,7 @@ The above snippet access the secrets `MY_ACCESS_KEY` and `MY_SECRET_KEY` previou Secrets **cannot** be assigned to pipeline parameters. ::: -## Process secrets +## Process directive Secrets can be access by pipeline processes by using the `secret` directive. For example: @@ -69,5 +69,22 @@ The secrets are made available in the process context running the command script ::: :::{note} -This feature is only available when using the local or grid executors (Slurm, Grid Engine, etc). The AWS Batch executor allows the use of secrets when deploying the pipeline execution via [Nextflow Tower](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/). +This feature is only available when using the local or grid executors (Slurm, Grid Engine, etc). The AWS Batch executor allows the use of secrets when deploying the pipeline execution via [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/). +::: + +## Pipeline script + +:::{versionadded} 24.03.0-edge +::: + +Secrets can be accessed in the pipeline script using the `secrets` variable. For example: + +```groovy +workflow.onComplete { + println("The secret is: ${secrets.MY_SECRET}") +} +``` + +:::{note} +This feature is only available when using the local or grid executors (Slurm, Grid Engine, etc). The AWS Batch executor allows the use of secrets when deploying the pipeline execution via [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/). ::: diff --git a/docs/snippets/process-out-env.nf b/docs/snippets/process-out-env.nf new file mode 100644 index 0000000000..848d2f9f82 --- /dev/null +++ b/docs/snippets/process-out-env.nf @@ -0,0 +1,13 @@ +process myTask { + output: + env FOO + + script: + ''' + FOO=$(seq 5) + ''' +} + +workflow { + myTask | view +} diff --git a/docs/snippets/process-out-env.out b/docs/snippets/process-out-env.out new file mode 100644 index 0000000000..85954eabcf --- /dev/null +++ b/docs/snippets/process-out-env.out @@ -0,0 +1,5 @@ +1 +2 +3 +4 +5 \ No newline at end of file diff --git a/docs/snippets/process-out-eval.nf b/docs/snippets/process-out-eval.nf new file mode 100644 index 0000000000..58081aafc7 --- /dev/null +++ b/docs/snippets/process-out-eval.nf @@ -0,0 +1,12 @@ +process sayHello { + output: + eval('bash --version') + + """ + echo Hello world! + """ +} + +workflow { + sayHello | view +} diff --git a/docs/snippets/process-out-eval.out b/docs/snippets/process-out-eval.out new file mode 100644 index 0000000000..9c08b97a7a --- /dev/null +++ b/docs/snippets/process-out-eval.out @@ -0,0 +1,6 @@ +GNU bash, version 5.1.16(1)-release (x86_64-pc-linux-gnu) +Copyright (C) 2020 Free Software Foundation, Inc. +License GPLv3+: GNU GPL version 3 or later + +This is free software; you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. \ No newline at end of file diff --git a/docs/snippets/process-stdout.nf b/docs/snippets/process-stdout.nf new file mode 100644 index 0000000000..9e2e719896 --- /dev/null +++ b/docs/snippets/process-stdout.nf @@ -0,0 +1,12 @@ +process sayHello { + output: + stdout + + """ + echo Hello world! + """ +} + +workflow { + sayHello | view { "I say... $it" } +} \ No newline at end of file diff --git a/docs/snippets/process-stdout.out b/docs/snippets/process-stdout.out new file mode 100644 index 0000000000..6df702c6e8 --- /dev/null +++ b/docs/snippets/process-stdout.out @@ -0,0 +1,2 @@ +I say... Hello world! + diff --git a/docs/sync-doc.sh b/docs/sync-doc.sh deleted file mode 100644 index af40b42309..0000000000 --- a/docs/sync-doc.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -TARGET=../../nextflow-website/assets -if grep -E "^release = '.*edge|.*SNAPSHOT'$" -c conf.py >/dev/null; then -MODE=edge -else -MODE=latest -fi -LATEST=$TARGET/docs/$MODE/ - -mkdir -p $LATEST - -rsync -r _build/html/ $LATEST - -( cd ../../nextflow-website; ./jbake) diff --git a/docs/wave.md b/docs/wave.md index f632850a3b..8d5f7536b8 100644 --- a/docs/wave.md +++ b/docs/wave.md @@ -38,7 +38,7 @@ tower { ``` :::{note} -The Tower access token is not mandatory, but it is recommended in order to access private container repositories and pull public containers without being affected by service rate limits. Credentials should be made available to Wave using the [credentials manager](https://help.tower.nf/latest/credentials/overview) in Tower. +The Seqera Platform access token is not mandatory, but it is recommended in order to access private container repositories and pull public containers without being affected by service rate limits. Credentials should be made available to Wave using the [credentials manager](https://docs.seqera.io/platform/latest/credentials/overview) in Seqera Platform. ::: ## Use cases @@ -47,9 +47,9 @@ The Tower access token is not mandatory, but it is recommended in order to acces ### Authenticate private repositories -Wave allows the use of private repositories in your Nextflow pipelines. The repository access keys must be provided in the form of [Tower credentials](https://help.tower.nf/latest/credentials/overview/). +Wave allows the use of private repositories in your Nextflow pipelines. The repository access keys must be provided in the form of [Seqera Platform credentials](https://docs.seqera.io/platform/latest/credentials/overview/). -Once the credentials have been created, simply specify your [Tower account access token](https://help.tower.nf/22.2/api/overview/#authentication) in your pipeline configuration file. If the credentials were created in a Tower organization workspace, specify the workspace ID as well in the config file as shown below: +Once the credentials have been created, simply specify your [personal access token](https://docs.seqera.io/platform/23.3.0/api/overview#authentication) in your pipeline configuration file. If the credentials were created in a Seqera Platform organization workspace, specify the workspace ID as well in the config file as shown below: ```groovy tower { @@ -99,8 +99,8 @@ Some configuration options in the `conda` scope are used when Wave is used to bu For example, the Conda channels and their priority can be set with `conda.channels`: ```groovy -wave.strategy = ['conda'] -conda.channels = 'seqera,conda-forge,bioconda,defaults' +wave.strategy = 'conda' +conda.channels = 'conda-forge,bioconda' ``` ::: @@ -159,7 +159,7 @@ In the above configuration replace `docker.io/user/repo` with a repository of yo should be uploaded. :::{note} -When using a private repository, the repository access keys must be provider via Tower credentials manager (see {ref}`above `). +When using a private repository, the repository access keys must be provided via the Seqera Platform credentials manager (see {ref}`above `). Moreover the access to the repository must be granted in the compute nodes by using the command `singularity remote login `. Please see Singularity documentation for further details. @@ -178,7 +178,7 @@ wave.build.cacheRepository = 'example.com/your/cache-repo' The first repository is used to store the built container images. The second one is used to store the individual image layers for caching purposes. -The repository access keys must be provided as Tower credentials (see +The repository access keys must be provided as Seqera Platform credentials (see [Authenticate private repositories](#authenticate-private-repositories) above). ### Run pipelines using Fusion file system @@ -191,71 +191,4 @@ See the {ref}`Fusion documentation ` for more details. ## Advanced settings -The following configuration options are available: - -`wave.enabled` -: Enable/disable the execution of Wave containers. - -`wave.endpoint` -: The Wave service endpoint (default: `https://wave.seqera.io`). - -`wave.freeze` -: :::{versionadded} 23.07.0-edge - ::: -: When enabling the container freeze mode, Wave will provision an non-ephemeral container image -that will be pushed to a container repository your choice. It requires the use of the `wave.build.repository` setting. -It is also suggested to specify a custom cache repository via the setting `wave.build.cacheRepository`. Note: when using -container freeze mode, the container repository authentication needs to be managed by the underlying infrastructure. - -`wave.build.repository` -: The container repository where images built by Wave are uploaded (note: the corresponding credentials must be provided in your Nextflow Tower account). - -`wave.build.cacheRepository` -: The container repository used to cache image layers built by the Wave service (note: the corresponding credentials must be provided in your Nextflow Tower account). - -`wave.build.conda.basePackages` -: One or more Conda packages to be always added in the resulting container (default: `conda-forge::procps-ng`). - -`wave.build.conda.commands` -: One or more commands to be added to the Dockerfile used to build a Conda based image. - -`wave.build.conda.mambaImage` -: The Mamba container image is used to build Conda based container. This is expected to be [micromamba-docker](https://github.com/mamba-org/micromamba-docker) image. - -`wave.build.spack.basePackages` -: :::{versionadded} 22.06.0-edge - ::: -: One or more Spack packages to be always added in the resulting container. - -`wave.build.spack.commands` -: :::{versionadded} 22.06.0-edge - ::: -: One or more commands to be added to the Dockerfile used to build a Spack based image. - -`wave.httpClient.connectTime` -: :::{versionadded} 22.06.0-edge - ::: -: Sets the connection timeout duration for the HTTP client connecting to the Wave service (default: `30s`). - -`wave.strategy` -: The strategy to be used when resolving ambiguous Wave container requirements (default: `'container,dockerfile,conda,spack'`). - -`wave.retryPolicy.delay` -: :::{versionadded} 22.06.0-edge - ::: -: The initial delay when a failing HTTP request is retried (default: `150ms`). - -`wave.retryPolicy.maxDelay` -: :::{versionadded} 22.06.0-edge - ::: -: The max delay when a failing HTTP request is retried (default: `90 seconds`). - -`wave.retryPolicy.maxAttempts` -: :::{versionadded} 22.06.0-edge - ::: -: The max number of attempts a failing HTTP request is retried (default: `5`). - -`wave.retryPolicy.jitter` -: :::{versionadded} 22.06.0-edge - ::: -: Sets the jitterFactor to randomly vary retry delays by (default: `0.25`). +Wave advanced configuration settings are described in the {ref}`Wave ` section on the Nextflow configuration page. diff --git a/docs/workflow.md b/docs/workflow.md index 6bc45f1015..98ddc34bc6 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -41,9 +41,115 @@ The `main:` label can be omitted if there are no `take:` or `emit:` blocks. Workflows were introduced in DSL2. If you are still using DSL1, see the {ref}`dsl1-page` page to learn how to migrate your Nextflow pipelines to DSL2. ::: +## Implicit workflow + +A script can define a single workflow without a name (also known as the *implicit workflow*), which is the default entrypoint of the script. The `-entry` command line option can be used to execute a different workflow as the entrypoint at runtime. + +:::{note} +Implicit workflow definitions are ignored when a script is included as a module. This way, a script can be written such that it can be either imported as a module or executed as a pipeline. +::: + +## Named workflows + +A named workflow is a workflow that can be invoked from other workflows. For example: + +```groovy +workflow my_pipeline { + foo() + bar( foo.out.collect() ) +} + +workflow { + my_pipeline() +} +``` + +The above snippet defines a workflow named `my_pipeline`, that can be invoked from another workflow as `my_pipeline()`, just like any other function or process. + +## Using variables and params + +A workflow can access any variable or parameter defined in the global scope: + +```groovy +params.data = '/some/data/file' + +workflow { + if( params.data ) + bar(params.data) + else + bar(foo()) +} +``` + +:::{tip} +The use of global variables and params in named workflows is discouraged because it breaks the modularity of the workflow. As a best practice, every workflow input should be explicitly defined as such in the `take:` block, and params should only be used in the implicit workflow. +::: + +## Workflow inputs (`take`) + +A workflow can declare one or more input channels using the `take` keyword. For example: + +```groovy +workflow my_pipeline { + take: + data1 + data2 + + main: + foo(data1, data2) + bar(foo.out) +} +``` + +:::{warning} +When the `take` keyword is used, the beginning of the workflow body must be defined with the `main` keyword. +::: + +Inputs can be specified like arguments when invoking the workflow: + +```groovy +workflow { + my_pipeline( channel.from('/some/data') ) +} +``` + +## Workflow outputs (`emit`) + +A workflow can declare one or more output channels using the `emit` keyword. For example: + +```groovy +workflow my_pipeline { + main: + foo(data) + bar(foo.out) + + emit: + bar.out +} +``` + +When invoking the workflow, the output channel(s) can be accessed using the `out` property, i.e. `my_pipeline.out`. When multiple output channels are declared, use the array bracket notation or the assignment syntax to access each output channel as described for [process outputs](#process-outputs). + +### Named outputs + +If an output channel is assigned to an identifier in the `emit` block, the identifier can be used to reference the channel from the calling workflow. For example: + +```groovy +workflow my_pipeline { + main: + foo(data) + bar(foo.out) + + emit: + my_data = bar.out +} +``` + +The result of the above workflow can be accessed using `my_pipeline.out.my_data`. + (workflow-process-invocation)= -## Process invocation +## Invoking processes A process can be invoked like a function in a workflow definition, passing the expected input channels like function arguments. For example: @@ -116,7 +222,7 @@ workflow { } ``` -### Process named outputs +#### Named outputs The `emit` option can be added to the process output definition to assign a name identifier. This name can be used to reference the channel from the calling workflow. For example: @@ -146,9 +252,10 @@ workflow { See {ref}`process outputs ` for more details. -### Process named stdout +#### Named stdout + +The `emit` option can also be used to name a `stdout` output. However, while process output options are usually prefixed with a comma, this is not the case for `stdout`. This is because `stdout` does not have an argument like other types. -The `emit` option can also be used to name a `stdout` output: ```groovy process sayHello { @@ -171,125 +278,7 @@ workflow { } ``` -:::{note} -Optional params for a process input/output are always prefixed with a comma, except for `stdout`. Because `stdout` does not have an associated name or value like other types, the first param should not be prefixed. -::: - -## Subworkflows - -A named workflow is a "subworkflow" that can be invoked from other workflows. For example: - -```groovy -workflow my_pipeline { - foo() - bar( foo.out.collect() ) -} - -workflow { - my_pipeline() -} -``` - -The above snippet defines a workflow named `my_pipeline`, that can be invoked from another workflow as `my_pipeline()`, just like any other function or process. - -### Workflow parameters - -A workflow component can access any variable or parameter defined in the global scope: - -```groovy -params.data = '/some/data/file' - -workflow my_pipeline { - if( params.data ) - bar(params.data) - else - bar(foo()) -} -``` - -### Workflow inputs - -A workflow can declare one or more input channels using the `take` keyword. For example: - -```groovy -workflow my_pipeline { - take: data - - main: - foo(data) - bar(foo.out) -} -``` - -Multiple inputs must be specified on separate lines: - -```groovy -workflow my_pipeline { - take: - data1 - data2 - - main: - foo(data1, data2) - bar(foo.out) -} -``` - -:::{warning} -When the `take` keyword is used, the beginning of the workflow body must be defined with the `main` keyword. -::: - -Inputs can be specified like arguments when invoking the workflow: - -```groovy -workflow { - my_pipeline( channel.from('/some/data') ) -} -``` - -### Workflow outputs - -A workflow can declare one or more output channels using the `emit` keyword. For example: - -```groovy -workflow my_pipeline { - main: - foo(data) - bar(foo.out) - - emit: - bar.out -} -``` - -When invoking the workflow, the output channel(s) can be accessed using the `out` property, i.e. `my_pipeline.out`. When multiple output channels are declared, use the array bracket notation or the assignment syntax to access each output channel as described for [process outputs](#process-outputs). - -### Workflow named outputs - -If an output channel is assigned to an identifier in the `emit` block, the identifier can be used to reference the channel from the calling workflow. For example: - -```groovy -workflow my_pipeline { - main: - foo(data) - bar(foo.out) - - emit: - my_data = bar.out -} -``` - -The result of the above workflow can be accessed using `my_pipeline.out.my_data`. - -### Workflow entrypoint - -A workflow with no name (also known as the *implicit workflow*) is the default entrypoint of the Nextflow pipeline. A different workflow entrypoint can be specified using the `-entry` command line option. - -:::{note} -Implicit workflow definitions are ignored when a script is included as a module. This way, a workflow script can be written in such a way that it can be used either as a library module or an application script. -::: - -### Workflow composition +## Invoking workflows Named workflows can be invoked and composed just like any other process or function. @@ -403,3 +392,291 @@ workflow { ``` In the above snippet, the initial channel is piped to the {ref}`operator-map` operator, which reverses the string value. Then, the result is passed to the processes `foo` and `bar`, which are executed in parallel. Each process outputs a channel, and the two channels are combined using the {ref}`operator-mix` operator. Finally, the result is printed using the {ref}`operator-view` operator. + +(workflow-output-def)= + +## Publishing outputs + +:::{versionadded} 24.04.0 +::: + +:::{note} +This feature requires the `nextflow.preview.output` feature flag to be enabled. +::: + +A script may define the set of outputs that should be published by the implicit workflow, known as the workflow output definition: + +```groovy +workflow { + foo(bar()) +} + +output { + directory 'results' +} +``` + +The output definition must be defined after the implicit workflow. + +### Publishing channels + +Processes and workflows can each define a `publish` section which maps channels to publish targets. For example: + +```groovy +process foo { + // ... + + output: + path 'result.txt', emit: results + + publish: + results >> 'foo/' + + // ... +} + +workflow foobar { + main: + foo(data) + bar(foo.out) + + publish: + foo.out >> 'foobar/foo/' + + emit: + bar.out +} +``` + +In the above example, the output `results` of process `foo` is published to the target `foo/` by default. However, when the workflow `foobar` invokes process `foo`, it publishes `foo.out` (i.e. `foo.out.results`) to the target `foobar/foo/`, overriding the default target defined by `foo`. + +In a process, any output with an `emit` name can be published. In a workflow, any channel defined in the workflow, including process and subworkflow outputs, can be published. + +:::{note} +If the publish source is a process/workflow output (e.g. `foo.out`) with multiple channels, each channel will be published. Individual output channels can also be published by index or name (e.g. `foo.out[0]` or `foo.out.results`). +::: + +As shown in the example, workflows can override the publish targets of process and subworkflow outputs. This way, each process and workflow can define some sensible defaults for publishing, which can be overridden by calling workflows as needed. + +By default, all files emitted by the channel will be published into the specified directory. If a channel emits list values, any files in the list (including nested lists) will also be published. For example: + +```groovy +workflow { + ch_samples = Channel.of( + [ [id: 'sample1'], file('sample1.txt') ] + ) + + publish: + ch_samples >> 'samples/' // sample1.txt will be published +} +``` + +### Publish directory + +The `directory` statement is used to set the top-level publish directory of the workflow: + +```groovy +output { + directory 'results' + + // ... +} +``` + +It is optional, and it defaults to the launch directory (`workflow.launchDir`). Published files will be saved within this directory. + +### Publish targets + +A publish target is a name with a specific publish configuration. By default, when a channel is published to a target in the `publish:` section of a process or workflow, the target name is used as the publish path. + +For example, given the following output definition: + +```groovy +workflow { + ch_foo = foo() + ch_bar = bar(ch_foo) + + publish: + ch_foo >> 'foo/' + ch_bar >> 'bar/' +} + +output { + directory 'results' +} +``` + +The following directory structure will be created: + +``` +results/ +└── foo/ + └── ... +└── bar/ + └── ... +``` + +:::{note} +The trailing slash in the target name is not required; it is only used to denote that the target name is intended to be used as the publish path. +::: + +:::{warning} +The target name must not begin with a slash (`/`), it should be a relative path name. +::: + +Workflows can also disable publishing for specific channels by redirecting them to `null`: + +```groovy +workflow { + ch_foo = foo() + + publish: + ch_foo >> (params.save_foo ? 'foo/' : null) +} +``` + +Publish targets can be customized in the output definition using a set of options similar to the {ref}`process-publishdir` directive. + +For example: + +```groovy +output { + directory 'results' + mode 'copy' + + 'foo/' { + mode 'link' + } +} +``` + +In this example, all files will be copied by default, and files published to `foo/` will be hard-linked, overriding the default option. + +Available options: + +`contentType` +: *Currently only supported for S3.* +: Specify the media type a.k.a. [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_Types) of published files (default: `false`). Can be a string (e.g. `'text/html'`), or `true` to infer the content type from the file extension. + +`enabled` +: Enable or disable publishing (default: `true`). + +`ignoreErrors` +: When `true`, the workflow will not fail if a file can't be published for some reason (default: `false`). + +`mode` +: The file publishing method (default: `'symlink'`). The following options are available: + + `'copy'` + : Copy each file into the output directory. + + `'copyNoFollow'` + : Copy each file into the output directory without following symlinks, i.e. only the link is copied. + + `'link'` + : Create a hard link in the output directory for each file. + + `'move'` + : Move each file into the output directory. + : Should only be used for files which are not used by downstream processes in the workflow. + + `'rellink'` + : Create a relative symbolic link in the output directory for each file. + + `'symlink'` + : Create an absolute symbolic link in the output directory for each output file. + +`overwrite` +: When `true` any existing file in the specified folder will be overwritten (default: `'standard'`). The following options are available: + + `false` + : Never overwrite existing files. + + `true` + : Always overwrite existing files. + + `'deep'` + : Overwrite existing files when the file content is different. + + `'lenient'` + : Overwrite existing files when the file size is different. + + `'standard'` + : Overwrite existing files when the file size or last modified timestamp is different. + +`path` +: Specify the publish path relative to the output directory (default: the target name). Can only be specified within a target definition. + +`storageClass` +: *Currently only supported for S3.* +: Specify the storage class for published files. + +`tags` +: *Currently only supported for S3.* +: Specify arbitrary tags for published files. For example: + ```groovy + tags FOO: 'hello', BAR: 'world' + ``` + +### Index files + +A publish target can create an index file of the values that were published. An index file is a useful way to save the metadata associated with files, and is more flexible than encoding metadata in the file path. Currently only CSV files are supported. + +For example: + +```groovy +workflow { + ch_foo = Channel.of( + [id: 1, name: 'foo 1'], + [id: 2, name: 'foo 2'], + [id: 3, name: 'foo 3'] + ) + + publish: + ch_foo >> 'foo/' +} + +output { + directory 'results' + + 'foo/' { + index { + path 'index.csv' + } + } +} +``` + +The above example will write the following CSV file to `results/foo/index.csv`: + +```csv +"id","name" +"1","foo 1" +"2","foo 2" +"3","foo 3" +``` + +You can customize the index file by specifying options in a block, for example: + +```groovy +index { + path 'index.csv' + header ['name', 'extra_option'] + sep '\t' + mapper { val -> val + [extra_option: 'bar'] } +} +``` + +The following options are available: + +`header` +: When `true`, the keys of the first record are used as the column names (default: `false`). Can also be a list of column names. + +`mapper` +: Closure which defines how to transform each published value into a CSV record. The closure should return a list or map. By default, no transformation is applied. + +`path` +: The name of the index file relative to the target path (required). + +`sep` +: The character used to separate values (default: `','`). diff --git a/docs/your-first-script.md b/docs/your-first-script.md new file mode 100644 index 0000000000..6b8cfcf10a --- /dev/null +++ b/docs/your-first-script.md @@ -0,0 +1,115 @@ +(your-first-script)= + +# Your first script + +## Run a pipeline + +This script defines two processes. The first splits a string into 6-character chunks, writing each one to a file with the prefix `chunk_`, and the second receives these files and transforms their contents to uppercase letters. The resulting strings are emitted on the `result` channel and the final output is printed by the `view` operator. Copy the following example into your favorite text editor and save it to a file named `tutorial.nf`: + +```{literalinclude} snippets/your-first-script.nf +:language: groovy +``` + +Execute the script by entering the following command in your terminal: + +```console +$ nextflow run tutorial.nf + +N E X T F L O W ~ version 23.10.0 +executor > local (3) +[69/c8ea4a] process > splitLetters [100%] 1 of 1 ✔ +[84/c8b7f1] process > convertToUpper [100%] 2 of 2 ✔ +HELLO +WORLD! +``` + +:::{note} +For versions of Nextflow prior to `22.10.0`, you must explicitly enable DSL2 by adding `nextflow.enable.dsl=2` to the top of the script or by using the `-dsl2` command-line option. +::: + +You can see that the first process is executed once, and the second twice. Finally the result string is printed. + +It's worth noting that the process `convertToUpper` is executed in parallel, so there's no guarantee that the instance processing the first split (the chunk `Hello`) will be executed before the one processing the second split (the chunk `world!`). Thus, you may very likely see the final result printed in a different order: + +``` +WORLD! +HELLO +``` + +:::{tip} +The hexadecimal string, e.g. `22/7548fa`, is the unique hash of a task, and the prefix of the directory where the task is executed. You can inspect a task's files by changing to the directory `$PWD/work` and using this string to find the specific task directory. +::: + +(getstarted-resume)= + +## Modify and resume + +Nextflow keeps track of all the processes executed in your pipeline. If you modify some parts of your script, only the processes that are actually changed will be re-executed. The execution of the processes that are not changed will be skipped and the cached result used instead. This helps a lot when testing or modifying part of your pipeline without having to re-execute it from scratch. + +For the sake of this tutorial, modify the `convertToUpper` process in the previous example, replacing the process script with the string `rev $x`, like so: + +```groovy +process convertToUpper { + input: + path x + output: + stdout + + """ + rev $x + """ +} +``` + +Then save the file with the same name, and execute it by adding the `-resume` option to the command line: + +```bash +nextflow run tutorial.nf -resume +``` + +It will print output similar to this: + +``` +N E X T F L O W ~ version 23.10.0 +executor > local (2) +[69/c8ea4a] process > splitLetters [100%] 1 of 1, cached: 1 ✔ +[d0/e94f07] process > convertToUpper [100%] 2 of 2 ✔ +olleH +!dlrow +``` + +You will see that the execution of the process `splitLetters` is actually skipped (the process ID is the same), and its results are retrieved from the cache. The second process is executed as expected, printing the reversed strings. + +:::{tip} +The pipeline results are cached by default in the directory `$PWD/work`. Depending on your script, this folder can take up a lot of disk space. It's a good idea to clean this folder periodically, as long as you know you won't need to resume any pipeline runs. +::: + +For more information, see the {ref}`cache-resume-page` page. + +(getstarted-params)= + +## Pipeline parameters + +Pipeline parameters are simply declared by prepending to a variable name the prefix `params`, separated by dot character. Their value can be specified on the command line by prefixing the parameter name with a double dash character, i.e. `--paramName` + +For the sake of this tutorial, you can try to execute the previous example specifying a different input string parameter, as shown below: + +```bash +nextflow run tutorial.nf --str 'Bonjour le monde' +``` + +The string specified on the command line will override the default value of the parameter. The output will look like this: + +``` +N E X T F L O W ~ version 23.10.0 +executor > local (4) +[8b/16e7d7] process > splitLetters [100%] 1 of 1 ✔ +[eb/729772] process > convertToUpper [100%] 3 of 3 ✔ +m el r +edno +uojnoB +``` + +:::{versionchanged} 20.11.0-edge +Any `.` (dot) character in a parameter name is interpreted as the delimiter of a nested scope. For example, `--foo.bar Hello` will be interpreted as `params.foo.bar`. If you want to have a parameter name that contains a `.` (dot) character, escape it using the back-slash character, e.g. `--foo\.bar Hello`. +::: diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e411586a54..48c0a02ca4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/launch.sh b/launch.sh index b910384518..818a8cd327 100755 --- a/launch.sh +++ b/launch.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2013-2023, Seqera Labs +# Copyright 2013-2024, Seqera Labs # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index bcc081b428..5080789bc0 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -17,19 +17,19 @@ compileGroovy { dependencies { api(project(':nf-commons')) api(project(':nf-httpfs')) - api "org.apache.groovy:groovy:4.0.18" - api "org.apache.groovy:groovy-nio:4.0.18" - api "org.apache.groovy:groovy-xml:4.0.18" - api "org.apache.groovy:groovy-json:4.0.18" - api "org.apache.groovy:groovy-templates:4.0.18" - api "org.apache.groovy:groovy-yaml:4.0.18" + api "org.apache.groovy:groovy:4.0.21" + api "org.apache.groovy:groovy-nio:4.0.21" + api "org.apache.groovy:groovy-xml:4.0.21" + api "org.apache.groovy:groovy-json:4.0.21" + api "org.apache.groovy:groovy-templates:4.0.21" + api "org.apache.groovy:groovy-yaml:4.0.21" api "org.slf4j:jcl-over-slf4j:2.0.7" api "org.slf4j:jul-to-slf4j:2.0.7" api "org.slf4j:log4j-over-slf4j:2.0.7" - api "ch.qos.logback:logback-classic:1.4.12" - api "ch.qos.logback:logback-core:1.4.12" + api "ch.qos.logback:logback-classic:1.4.14" + api "ch.qos.logback:logback-core:1.4.14" api "org.codehaus.gpars:gpars:1.2.1" - api("ch.artecat.grengine:grengine:3.0.0") { exclude group: 'org.codehaus.groovy' } + api("ch.artecat.grengine:grengine:3.0.2") { exclude group: 'org.codehaus.groovy' } api "commons-lang:commons-lang:2.6" api "commons-codec:commons-codec:1.15" api "commons-io:commons-io:2.11.0" @@ -39,7 +39,7 @@ dependencies { api('org.eclipse.jgit:org.eclipse.jgit:6.6.1.202309021850-r') api ('javax.activation:activation:1.1.1') api ('javax.mail:mail:1.4.7') - api ('org.yaml:snakeyaml:2.0') + api ('org.yaml:snakeyaml:2.2') api ('org.jsoup:jsoup:1.15.4') api 'jline:jline:2.9' api 'org.pf4j:pf4j:3.10.0' @@ -48,7 +48,7 @@ dependencies { testImplementation 'org.subethamail:subethasmtp:3.1.7' // test configuration - testFixturesApi ("org.apache.groovy:groovy-test:4.0.18") { exclude group: 'org.apache.groovy' } + testFixturesApi ("org.apache.groovy:groovy-test:4.0.21") { exclude group: 'org.apache.groovy' } testFixturesApi ("cglib:cglib-nodep:3.3.0") testFixturesApi ("org.objenesis:objenesis:3.2") testFixturesApi ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy'; exclude group: 'net.bytebuddy' } diff --git a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy index 84c0d2bf0e..6cef20e32b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/NF.groovy b/modules/nextflow/src/main/groovy/nextflow/NF.groovy index 3f13630088..4767e41d56 100644 --- a/modules/nextflow/src/main/groovy/nextflow/NF.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/NF.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -64,6 +64,10 @@ class NF { NextflowMeta.instance.isStrictModeEnabled() } + static boolean isOutputDefinitionEnabled() { + NextflowMeta.instance.preview.output + } + static boolean isRecurseEnabled() { NextflowMeta.instance.preview.recursion } diff --git a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy index 5976f229e3..3f1257a01c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy b/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy index bb836ab61a..c45d8cecfe 100644 --- a/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy @@ -37,14 +37,15 @@ class NextflowMeta { abstract boolean strict } - @Deprecated @Slf4j static class Preview implements Flags { - volatile float dsl - boolean strict + @Deprecated volatile float dsl + @Deprecated boolean strict + boolean output boolean recursion boolean topic + @Deprecated void setDsl( float num ) { if( num == 1 ) throw new IllegalArgumentException(DSL1_EOL_MESSAGE) @@ -55,16 +56,22 @@ class NextflowMeta { dsl = num } - void setRecursion(Boolean recurse) { - if( recurse ) + void setOutput(Boolean output) { + if( output ) + log.warn "WORKFLOW OUTPUT DSL IS A PREVIEW FEATURE - SYNTAX AND FUNCTIONALITY CAN CHANGE IN FUTURE RELEASES" + this.output = output + } + + void setRecursion(Boolean recursion) { + if( recursion ) log.warn "NEXTFLOW RECURSION IS A PREVIEW FEATURE - SYNTAX AND FUNCTIONALITY CAN CHANGE IN FUTURE RELEASES" - this.recursion = recurse + this.recursion = recursion } - void setTopic(Boolean value) { + void setTopic(Boolean topic) { if( topic ) log.warn "CHANNEL TOPICS ARE A PREVIEW FEATURE - SYNTAX AND FUNCTIONALITY CAN CHANGE IN FUTURE RELEASES" - this.topic = value + this.topic = topic } } @@ -81,7 +88,6 @@ class NextflowMeta { */ final String timestamp - @Deprecated final Preview preview = new Preview() final Features enable = new Features() diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 48713ea603..bb4f6651fc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package nextflow -import static nextflow.Const.* import java.nio.file.Files import java.nio.file.Path @@ -32,6 +31,7 @@ import groovy.transform.Memoized import groovy.transform.PackageScope import groovy.util.logging.Slf4j import groovyx.gpars.GParsConfig +import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.cache.CacheDB import nextflow.cache.CacheFactory @@ -48,8 +48,6 @@ import nextflow.executor.ExecutorFactory import nextflow.extension.CH import nextflow.file.FileHelper import nextflow.file.FilePorter -import nextflow.util.Threads -import nextflow.util.ThreadPoolManager import nextflow.plugin.Plugins import nextflow.processor.ErrorStrategy import nextflow.processor.TaskFault @@ -74,6 +72,8 @@ import nextflow.util.ConfigHelper import nextflow.util.Duration import nextflow.util.HistoryFile import nextflow.util.NameGenerator +import nextflow.util.ThreadPoolManager +import nextflow.util.Threads import nextflow.util.VersionNumber import org.apache.commons.lang.exception.ExceptionUtils import sun.misc.Signal @@ -94,6 +94,8 @@ class Session implements ISession { final List igniters = new ArrayList<>(20) + final Map publishTargets = [:] + /** * Creates process executors */ @@ -154,6 +156,11 @@ class Session implements ISession { */ boolean stubRun + /** + * Enable preview mode + */ + boolean preview + /** * Folder(s) containing libs and classes to be added to the classpath */ @@ -346,6 +353,9 @@ class Session implements ISession { // -- dry run this.stubRun = config.stubRun + // -- preview + this.preview = config.preview + // -- normalize taskConfig object if( config.process == null ) config.process = [:] if( config.env == null ) config.env = [:] @@ -666,8 +676,9 @@ class Session implements ISession { void destroy() { try { log.trace "Session > destroying" - // shutdown publish dir executor - publishPoolManager.shutdown(aborted) + // shutdown thread pools + finalizePoolManager?.shutdown(aborted) + publishPoolManager?.shutdown(aborted) // invoke shutdown callbacks shutdown0() log.trace "Session > after cleanup" @@ -682,9 +693,6 @@ class Session implements ISession { // -- close db cache?.close() - // -- shutdown plugins - Plugins.stop() - // -- cleanup script classes dir classesDir?.deleteDir() } @@ -1431,10 +1439,25 @@ class Session implements ISession { ansiLogObserver ? ansiLogObserver.appendInfo(file.text) : Files.copy(file, System.out) } - private ThreadPoolManager publishPoolManager = new ThreadPoolManager('PublishDir') + private volatile ThreadPoolManager finalizePoolManager + + @Memoized + synchronized ExecutorService taskFinalizerExecutorService() { + finalizePoolManager = new ThreadPoolManager('TaskFinalizer') + return finalizePoolManager + .withConfig(config) + .withShutdownMessage( + "Waiting for remaining tasks to complete (%d tasks)", + "Exiting before some tasks were completed" + ) + .create() + } + + private volatile ThreadPoolManager publishPoolManager @Memoized synchronized ExecutorService publishDirExecutorService() { + publishPoolManager = new ThreadPoolManager('PublishDir') return publishPoolManager .withConfig(config) .create() diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/ASTHelpers.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/ASTHelpers.groovy index a03db18790..595cdc67eb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/ASTHelpers.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/ASTHelpers.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/GStringToLazyVisitor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/GStringToLazyVisitor.groovy index e332f05df5..c52fa0bd0f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/GStringToLazyVisitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/GStringToLazyVisitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/LangHelpers.java b/modules/nextflow/src/main/groovy/nextflow/ast/LangHelpers.java index 9fc9b04589..19746c463d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/LangHelpers.java +++ b/modules/nextflow/src/main/groovy/nextflow/ast/LangHelpers.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy index 7d54cb0e8f..d51cae138b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSL.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy index 0586019d7a..b3c16b4af7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import nextflow.script.BaseScript import nextflow.script.BodyDef import nextflow.script.IncludeDef import nextflow.script.TaskClosure +import nextflow.script.TokenEvalCall import nextflow.script.TokenEnvCall import nextflow.script.TokenFileCall import nextflow.script.TokenPathCall @@ -81,7 +82,8 @@ class NextflowDSLImpl implements ASTTransformation { final static private String WORKFLOW_TAKE = 'take' final static private String WORKFLOW_EMIT = 'emit' final static private String WORKFLOW_MAIN = 'main' - final static private List SCOPES = [WORKFLOW_TAKE, WORKFLOW_EMIT, WORKFLOW_MAIN] + final static private String WORKFLOW_PUBLISH = 'publish' + final static private List SCOPES = [WORKFLOW_TAKE, WORKFLOW_EMIT, WORKFLOW_MAIN, WORKFLOW_PUBLISH] final static public String PROCESS_WHEN = 'when' final static public String PROCESS_STUB = 'stub' @@ -171,11 +173,17 @@ class NextflowDSLImpl implements ASTTransformation { currentTaskName = null } } + else if( methodName == 'workflow' && preCondition ) { convertWorkflowDef(methodCall,sourceUnit) super.visitMethodCallExpression(methodCall) } + else if( methodName == 'output' && preCondition ) { + convertOutputDef(methodCall,sourceUnit) + super.visitMethodCallExpression(methodCall) + } + // just apply the default behavior else { super.visitMethodCallExpression(methodCall) @@ -422,6 +430,21 @@ class NextflowDSLImpl implements ASTTransformation { return result } + protected Statement normWorkflowPublish(ExpressionStatement stm) { + if( stm.expression !instanceof BinaryExpression ) { + syntaxError(stm, "Invalid workflow publish statement") + return stm + } + + final binaryX = (BinaryExpression)stm.expression + if( binaryX.operation.type != Types.RIGHT_SHIFT ) { + syntaxError(stm, "Invalid workflow publish statement") + return stm + } + + return stmt( callThisX('_publish_target', args(binaryX.leftExpression, binaryX.rightExpression)) ) + } + protected Expression makeWorkflowDefWrapper( ClosureExpression closure, boolean anonymous ) { final codeBlock = (BlockStatement) closure.code @@ -461,6 +484,14 @@ class NextflowDSLImpl implements ASTTransformation { body.add(stm) break + case WORKFLOW_PUBLISH: + if( !(stm instanceof ExpressionStatement) ) { + syntaxError(stm, "Invalid workflow publish statement") + break + } + body.add(normWorkflowPublish(stm as ExpressionStatement)) + break + default: if( context ) { def opts = SCOPES.closest(context) @@ -472,7 +503,7 @@ class NextflowDSLImpl implements ASTTransformation { } } // read the closure source - readSource(closure, source, unit, true) + readSource(closure, source, unit) final bodyClosure = closureX(null, block(scope, body)) final invokeBody = makeScriptWrapper(bodyClosure, source.toString(), 'workflow', unit) @@ -487,6 +518,62 @@ class NextflowDSLImpl implements ASTTransformation { unit.addError( new SyntaxException(message,line,coln)) } + /** + * Transform targets in the workflow output definition: + * + * output { + * 'foo' { ... } + * } + * + * becomes: + * + * output { + * target('foo') { ... } + * } + * + * @param methodCall + * @param unit + */ + protected void convertOutputDef(MethodCallExpression methodCall, SourceUnit unit) { + log.trace "Convert 'output' ${methodCall.arguments}" + + assert methodCall.arguments instanceof ArgumentListExpression + final arguments = (ArgumentListExpression)methodCall.arguments + + if( arguments.size() != 1 || arguments[0] !instanceof ClosureExpression ) { + syntaxError(methodCall, "Invalid output definition") + return + } + + final closure = (ClosureExpression)arguments[0] + final block = (BlockStatement)closure.code + for( Statement stmt : block.statements ) { + if( stmt !instanceof ExpressionStatement ) { + syntaxError(stmt, "Invalid publish target definition") + return + } + + final stmtExpr = (ExpressionStatement)stmt + if( stmtExpr.expression !instanceof MethodCallExpression ) { + syntaxError(stmt, "Invalid publish target definition") + return + } + + final call = (MethodCallExpression)stmtExpr.expression + assert call.arguments instanceof ArgumentListExpression + + // HACK: target definition is a method call with single closure argument + // custom parser will be able to detect more elegantly + final targetArgs = (ArgumentListExpression)call.arguments + if( targetArgs.size() != 1 || targetArgs[0] !instanceof ClosureExpression ) + continue + + final targetName = call.method + final targetBody = (ClosureExpression)targetArgs[0] + stmtExpr.expression = callThisX('target', args(targetName, targetBody)) + } + } + /** * Transform a DSL `process` definition into a proper method invocation * @@ -546,6 +633,11 @@ class NextflowDSLImpl implements ASTTransformation { } break + case 'publish': + if( stm instanceof ExpressionStatement ) + convertPublishMethod( stm ) + break + case 'exec': bodyLabel = currentLabel iterator.remove() @@ -749,7 +841,29 @@ class NextflowDSLImpl implements ASTTransformation { * @param buffer * @param unit */ - private void readSource( ASTNode node, StringBuilder buffer, SourceUnit unit, stripBrackets=false ) { + private void readSource( Statement node, StringBuilder buffer, SourceUnit unit ) { + final colx = node.getColumnNumber() + final colz = node.getLastColumnNumber() + final first = node.getLineNumber() + final last = node.getLastLineNumber() + for( int i = first; i <= last; i++ ) { + final line = unit.source.getLine(i, null) + + // prepend first-line indent + if( i == first ) { + int k = 0 + while( k < line.size() && line[k] == ' ' ) + k++ + buffer.append( line.substring(0, k) ) + } + + final begin = (i == first) ? colx - 1 : 0 + final end = (i == last) ? colz - 1 : line.size() + buffer.append( line.substring(begin, end) ).append('\n') + } + } + + private void readSource( ClosureExpression node, StringBuilder buffer, SourceUnit unit ) { final colx = node.getColumnNumber() final colz = node.getLastColumnNumber() final first = node.getLineNumber() @@ -757,18 +871,12 @@ class NextflowDSLImpl implements ASTTransformation { for( int i=first; i<=last; i++ ) { def line = unit.source.getLine(i, null) if( i==last ) { - line = line.substring(0,colz-1) - if( stripBrackets ) { - line = line.replaceFirst(/}.*$/,'') - if( !line.trim() ) continue - } + line = line.substring(0,colz-1).replaceFirst(/}.*$/,'') + if( !line.trim() ) continue } if( i==first ) { - line = line.substring(colx-1) - if( stripBrackets ) { - line = line.replaceFirst(/^.*\{/,'').trim() - if( !line.trim() ) continue - } + line = line.substring(colx-1).replaceFirst(/^.*\{/,'').trim() + if( !line ) continue } buffer.append(line) .append('\n') } @@ -955,7 +1063,7 @@ class NextflowDSLImpl implements ASTTransformation { def nested = methodCall.objectExpression instanceof MethodCallExpression log.trace "convert > output method: $methodName" - if( methodName in ['val','env','file','set','stdout','path','tuple'] && !nested ) { + if( methodName in ['val','env','eval','file','set','stdout','path','tuple'] && !nested ) { // prefix the method name with the string '_out_' methodCall.setMethod( new ConstantExpression('_out_' + methodName) ) fixMethodCall(methodCall) @@ -1123,6 +1231,11 @@ class NextflowDSLImpl implements ASTTransformation { return createX( TokenEnvCall, args ) } + if( methodCall.methodAsString == 'eval' && withinTupleMethod ) { + def args = (TupleExpression) varToStrX(methodCall.arguments) + return createX( TokenEvalCall, args ) + } + /* * input: * tuple val(x), .. from q @@ -1186,6 +1299,27 @@ class NextflowDSLImpl implements ASTTransformation { return false } + protected void convertPublishMethod(ExpressionStatement stmt) { + if( stmt.expression !instanceof BinaryExpression ) { + syntaxError(stmt, "Invalid process publish statement") + return + } + + final binaryX = (BinaryExpression)stmt.expression + if( binaryX.operation.type != Types.RIGHT_SHIFT ) { + syntaxError(stmt, "Invalid process publish statement") + return + } + + final left = binaryX.leftExpression + if( left !instanceof VariableExpression ) { + syntaxError(stmt, "Invalid process publish statement") + return + } + + stmt.expression = callThisX('_publish_target', args(constX(((VariableExpression)left).name), binaryX.rightExpression)) + } + protected boolean isIllegalName(String name, ASTNode node) { if( name in RESERVED_NAMES ) { unit.addError( new SyntaxException("Identifier `$name` is reserved for internal use", node.lineNumber, node.columnNumber+8) ) diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXform.groovy index 5eccda1283..d76b86bceb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXform.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXform.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy index e4359118d3..8fbf71d6e7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowXformImpl.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy index 76c6d86ba6..892ded9aed 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/OpXform.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy index c60a2a341e..cded4e89f5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/OpXformImpl.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXform.groovy index 954d597a49..c35d605970 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXform.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXform.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXformVisitor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXformVisitor.groovy index 683b331cff..9a7d37927f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXformVisitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/TaskCmdXformVisitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXform.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXform.groovy index 39cf32d8aa..28e3dd1b1d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXform.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXform.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXformImpl.groovy index 4be53993bc..e5ee8eb120 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVarsXformImpl.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVisitor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVisitor.groovy index aaae7c2fb5..63b8fa6e80 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVisitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/TaskTemplateVisitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/VariableVisitor.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/VariableVisitor.groovy index 758ea5f854..71c453054b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/VariableVisitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/VariableVisitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy b/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy index 214e3235e3..6e5c9d755b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cache/CacheDB.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cache/CacheFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/cache/CacheFactory.groovy index 4623243cdd..84d020a795 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cache/CacheFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cache/CacheFactory.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cache/CacheStore.groovy b/modules/nextflow/src/main/groovy/nextflow/cache/CacheStore.groovy index 02ea743955..452e7fcf4f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cache/CacheStore.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cache/CacheStore.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheFactory.groovy index 2b4c0387e6..8f1f2682d9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheFactory.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheStore.groovy b/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheStore.groovy index 66fb43a91e..effbf8b329 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheStore.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cache/DefaultCacheStore.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,10 +18,10 @@ package nextflow.cache import java.nio.file.Path -import java.nio.file.Paths import com.google.common.hash.HashCode import groovy.transform.CompileStatic +import nextflow.Const import nextflow.exception.AbortOperationException import nextflow.util.CacheHelper import org.iq80.leveldb.DB @@ -64,7 +64,7 @@ class DefaultCacheStore implements CacheStore { this.KEY_SIZE = CacheHelper.hasher('x').hash().asBytes().size() this.uniqueId = uniqueId this.runName = runName - this.baseDir = home ?: Paths.get('.nextflow').toAbsolutePath() + this.baseDir = home ?: Const.appCacheDir.toAbsolutePath() this.dataDir = baseDir.resolve("cache/$uniqueId") this.indexFile = dataDir.resolve("index.$runName") } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CacheBase.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CacheBase.groovy index a6f7847b47..f96f854a1d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CacheBase.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CacheBase.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,7 @@ trait CacheBase { void init() { if( !history ) { - history = !basePath ? HistoryFile.DEFAULT : new HistoryFile(basePath.resolve(HistoryFile.FILE_NAME)) + history = !basePath ? HistoryFile.DEFAULT : new HistoryFile(basePath.resolve(HistoryFile.defaultFileName())) } if( !history.exists() || history.empty() ) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CliOptions.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CliOptions.groovy index 4781af73c2..172790edfc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CliOptions.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CliOptions.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy index 79efc23932..75b4cde37b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy index 48be429229..bb58edacbb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy index 28c1d209c3..9c582aaaf6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy index 29b874198b..5ff2156b18 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,6 +89,7 @@ class CmdConfig extends CmdBase { final builder = new ConfigBuilder() .setShowClosures(true) + .setStripSecrets(true) .showMissingVariables(true) .setOptions(launcher.options) .setBaseDir(base) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConsole.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConsole.groovy index afd58640aa..6cd57aff6d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConsole.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdConsole.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdDrop.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdDrop.groovy index dd36368854..9d67190a54 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdDrop.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdDrop.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy index 65320890b1..bbf11dfb0f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelp.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelp.groovy index 5a42bc9a98..80d2b215fa 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelper.groovy index 49906bd8a2..5e41748d0a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelper.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy index e679d1fcd5..c8c1c31bc0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInspect.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdInspect.groovy index c865c8aee8..21582fc692 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInspect.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdInspect.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -85,6 +85,7 @@ class CmdInspect extends CmdBase { target.preview = true target.previewAction = this.&applyInspect target.ansiLog = false + target.skipHistoryFile = true // run it target.run() } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy index fe39c18e2d..38ebb9e695 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdList.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdList.groovy index 9eda7b8c66..5b08be249a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdList.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdList.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLog.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdLog.groovy index 22a6efb91c..66d88980d1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLog.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdLog.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy index 43339652ec..47666166d6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPlugin.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdPlugin.groovy index 38414fc052..370508f49b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPlugin.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdPlugin.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,6 +47,8 @@ class CmdPlugin extends CmdBase { throw new AbortOperationException("Missing plugin command - usage: nextflow plugin install ") // setup plugins system Plugins.init() + Runtime.addShutdownHook((it)-> Plugins.stop()) + // check for the plugins install if( args[0] == 'install' ) { if( args.size()!=2 ) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy index 24f7d552d4..9d2fa9a01d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index 98b90f5d01..e5d18bdb87 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package nextflow.cli +import static org.fusesource.jansi.Ansi.* + import java.nio.file.NoSuchFileException import java.nio.file.Path import java.util.regex.Pattern @@ -41,10 +43,10 @@ import nextflow.plugin.Plugins import nextflow.scm.AssetManager import nextflow.script.ScriptFile import nextflow.script.ScriptRunner -import nextflow.secret.SecretsLoader import nextflow.util.CustomPoolFactory import nextflow.util.Duration import nextflow.util.HistoryFile +import org.fusesource.jansi.AnsiConsole import org.yaml.snakeyaml.Yaml /** * CLI sub-command RUN @@ -60,8 +62,8 @@ class CmdRun extends CmdBase implements HubOptions { static final public List VALID_PARAMS_FILE = ['json', 'yml', 'yaml'] - static final public DSL2 = '2' - static final public DSL1 = '1' + static final public String DSL2 = '2' + static final public String DSL1 = '1' static { // install the custom pool factory for GPars threads @@ -160,7 +162,7 @@ class CmdRun extends CmdBase implements HubOptions { launcher.options.ansiLog = value } - @Parameter(names = ['-with-tower'], description = 'Monitor workflow execution with Seqera Tower service') + @Parameter(names = ['-with-tower'], description = 'Monitor workflow execution with Seqera Platform (formerly Tower Cloud)') String withTower @Parameter(names = ['-with-wave'], hidden = true) @@ -264,6 +266,8 @@ class CmdRun extends CmdBase implements HubOptions { @Parameter(names=['-disable-jobs-cancellation'], description = 'Prevent the cancellation of child jobs on execution termination') Boolean disableJobsCancellation + Boolean skipHistoryFile + Boolean getDisableJobsCancellation() { return disableJobsCancellation!=null ? disableJobsCancellation @@ -310,7 +314,7 @@ class CmdRun extends CmdBase implements HubOptions { checkRunName() - log.info "N E X T F L O W ~ version ${BuildInfo.version}" + printBanner() Plugins.init() // -- specify the arguments @@ -333,12 +337,6 @@ class CmdRun extends CmdBase implements HubOptions { final cfg = plugins ? [plugins: plugins.tokenize(',')] : config Plugins.load(cfg) - // -- load secret provider - if( SecretsLoader.isEnabled() ) { - final provider = SecretsLoader.instance.load() - config.withSecretProvider(provider) - } - // -- create a new runner instance final runner = new ScriptRunner(config) runner.setScript(scriptFile) @@ -366,12 +364,45 @@ class CmdRun extends CmdBase implements HubOptions { log.debug( '\n'+info ) // -- add this run to the local history - runner.verifyAndTrackHistory(launcher.cliString, runName) + if( !skipHistoryFile ) { + runner.verifyAndTrackHistory(launcher.cliString, runName) + } // -- run it! runner.execute(scriptArgs, this.entryName) } + protected void printBanner() { + if( launcher.options.ansiLog ){ + // Plain header for verbose log + log.debug "N E X T F L O W ~ version ${BuildInfo.version}" + + // Fancy coloured header for the ANSI console output + def fmt = ansi() + fmt.a("\n") + // Use exact colour codes so that they render the same on every terminal, + // irrespective of terminal colour scheme. + // Nextflow green RGB (13, 192, 157) and exact black text (0,0,0), + // Apple Terminal only supports 256 colours, so use the closest match: + // light sea green | #20B2AA | 38;5;0 + // Don't use black for text as terminals mess with this in their colour schemes. + // Use very dark grey, which is more reliable. + // Jansi library bundled in Jline can't do exact RGBs, + // so just do the ANSI codes manually + final BACKGROUND = "\033[1m\033[38;5;232m\033[48;5;43m" + fmt.a("$BACKGROUND N E X T F L O W ").reset() + + // Show Nextflow version + fmt.a(Attribute.INTENSITY_FAINT).a(" ~ ").reset().a("version " + BuildInfo.version).reset() + fmt.a("\n") + AnsiConsole.out.println(fmt.eraseLine()) + } + else { + // Plain header to the console if ANSI is disabled + log.info "N E X T F L O W ~ version ${BuildInfo.version}" + } + } + protected checkConfigEnv(ConfigMap config) { // Warn about setting NXF_ environment variables within env config scope final env = config.env as Map @@ -396,12 +427,32 @@ class CmdRun extends CmdBase implements HubOptions { NextflowMeta.instance.enableDsl(dsl) // -- show launch info final ver = NF.dsl2 ? DSL2 : DSL1 - final repo = scriptFile.repository ?: scriptFile.source + final repo = scriptFile.repository ?: scriptFile.source.toString() final head = preview ? "* PREVIEW * $scriptFile.repository" : "Launching `$repo`" - if( scriptFile.repository ) - log.info "${head} [$runName] DSL${ver} - revision: ${scriptFile.revisionInfo}" - else - log.info "${head} [$runName] DSL${ver} - revision: ${scriptFile.getScriptId()?.substring(0,10)}" + final revision = scriptFile.repository + ? scriptFile.revisionInfo.toString() + : scriptFile.getScriptId()?.substring(0,10) + printLaunchInfo(ver, repo, head, revision) + } + + protected void printLaunchInfo(String ver, String repo, String head, String revision) { + if( launcher.options.ansiLog ){ + log.debug "${head} [$runName] DSL${ver} - revision: ${revision}" + + def fmt = ansi() + fmt.a("Launching").fg(Color.MAGENTA).a(" `$repo` ").reset() + fmt.a(Attribute.INTENSITY_FAINT).a("[").reset() + fmt.bold().fg(Color.CYAN).a(runName).reset() + fmt.a(Attribute.INTENSITY_FAINT).a("]") + fmt.a(" DSL${ver} - ") + fmt.fg(Color.CYAN).a("revision: ").reset() + fmt.fg(Color.CYAN).a(revision).reset() + fmt.a("\n") + AnsiConsole.out().println(fmt.eraseLine()) + } + else { + log.info "${head} [$runName] DSL${ver} - revision: ${revision}" + } } static String detectDslMode(ConfigMap config, String scriptText, Map sysEnv) { diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdSecret.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdSecret.groovy index 51b6e879c7..cb4aa60acf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdSecret.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdSecret.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy index 91718ab447..475e6b3ca0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdView.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdView.groovy index d38befaf07..dcdcfcae67 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdView.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdView.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy index 7daa1caffd..9a022afda1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy index f99702b90a..64531ae50e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +41,9 @@ import nextflow.util.LoggerHelper import nextflow.util.ProxyConfig import nextflow.util.SpuriousDeps import org.eclipse.jgit.api.errors.GitAPIException + +import static nextflow.util.SysHelper.dumpThreads + /** * Main application entry point. It parses the command line and * launch the pipeline execution. @@ -553,24 +556,6 @@ class Launcher { } - /** - * Dump th stack trace of current running threads - * @return - */ - private String dumpThreads() { - - def buffer = new StringBuffer() - Map m = Thread.getAllStackTraces(); - for(Map.Entry e : m.entrySet()) { - buffer.append('\n').append(e.getKey().toString()).append('\n') - for (StackTraceElement s : e.getValue()) { - buffer.append(" " + s).append('\n') - } - } - - return buffer.toString() - } - /** * set up environment and system properties. It checks the following * environment variables: diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy index 620259f4fa..8cb1391b00 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/PluginExecAware.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/PluginExecAware.groovy index 91bcf48407..0357854424 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/PluginExecAware.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/PluginExecAware.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/usageAware.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/usageAware.groovy index e12d3bdb3d..e2dbd99923 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/usageAware.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/usageAware.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/CloudSpotTerminationException.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/CloudSpotTerminationException.groovy index 512bf140b5..39a7f72b03 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/CloudSpotTerminationException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/CloudSpotTerminationException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstance.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstance.groovy index f3fa6f6864..4b602cf3af 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstance.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstance.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceStatus.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceStatus.groovy index a7a6884abf..6a4aec4ffb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceStatus.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceStatus.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceType.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceType.groovy index 6097a99d26..6d38ef3fbc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceType.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudInstanceType.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudMachineInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudMachineInfo.groovy index 2093b55ffc..32d3b823a5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudMachineInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudMachineInfo.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudSpotPrice.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudSpotPrice.groovy index 58633d9976..5e5765f547 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudSpotPrice.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/types/CloudSpotPrice.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/types/PriceModel.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/types/PriceModel.groovy index e1c844a35a..2444acfbf2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/types/PriceModel.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/types/PriceModel.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy index b02a17f669..6a008a6b6a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/conda/CondaConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/conda/CondaConfig.groovy index 6f471af5c4..ea79a435b2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/conda/CondaConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/conda/CondaConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/config/CascadingConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/config/CascadingConfig.groovy index f393e224b4..49f4ef0841 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/CascadingConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/CascadingConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy index 13031d950a..46a6f182da 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBase.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import groovy.transform.Memoized import java.nio.file.NoSuchFileException import java.nio.file.Path +import nextflow.exception.IllegalConfigException import nextflow.file.FileHelper import org.codehaus.groovy.control.CompilerConfiguration import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer @@ -49,6 +50,12 @@ abstract class ConfigBase extends Script { private boolean renderClosureAsString + private boolean stripSecrets + + protected void setStripSecrets( boolean value ) { + this.stripSecrets = value + } + protected void setIgnoreIncludes( boolean value ) { this.ignoreIncludes = value } @@ -71,7 +78,8 @@ abstract class ConfigBase extends Script { * Implements the config file include */ def includeConfig( includeFile ) { - assert includeFile + if( !includeFile ) + throw new IllegalConfigException("includeConfig argument cannot be empty") if( ignoreIncludes ) return @@ -92,6 +100,8 @@ abstract class ConfigBase extends Script { // -- set the required base script def config = new CompilerConfiguration() config.scriptBaseClass = ConfigBase.class.name + if( stripSecrets ) + config.addCompilationCustomizers(new ASTTransformationCustomizer(StripSecretsXform)) def params = [:] if( renderClosureAsString ) params.put('renderClosureAsString', true) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy index d30ad9e0dd..08d5f3f011 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,8 +32,6 @@ import nextflow.cli.CmdNode import nextflow.cli.CmdRun import nextflow.exception.AbortOperationException import nextflow.exception.ConfigParseException -import nextflow.secret.SecretHolder -import nextflow.secret.SecretsContext import nextflow.secret.SecretsLoader import nextflow.trace.GraphObserver import nextflow.trace.ReportObserver @@ -77,6 +75,8 @@ class ConfigBuilder { boolean showClosures + boolean stripSecrets + boolean showMissingVariables Map emptyVariables = new LinkedHashMap<>(10) @@ -95,6 +95,11 @@ class ConfigBuilder { return this } + ConfigBuilder setStripSecrets(boolean value) { + this.stripSecrets = value + return this + } + ConfigBuilder showMissingVariables(boolean value) { this.showMissingVariables = value return this @@ -333,8 +338,7 @@ class ConfigBuilder { binding.put('baseDir', base) binding.put('projectDir', base) binding.put('launchDir', Paths.get('.').toRealPath()) - if( SecretsLoader.isEnabled() ) - binding.put('secrets', new SecretsContext()) + binding.put('secrets', SecretsLoader.secretContext()) return binding } @@ -344,6 +348,7 @@ class ConfigBuilder { final ignoreIncludes = options ? options.ignoreConfigIncludes : false final slurper = new ConfigParser() .setRenderClosureAsString(showClosures) + .setStripSecrets(stripSecrets) .setIgnoreIncludes(ignoreIncludes) ConfigObject result = new ConfigObject() @@ -540,6 +545,9 @@ class ConfigBuilder { if( cmdRun.stubRun ) config.stubRun = cmdRun.stubRun + if( cmdRun.preview ) + config.preview = cmdRun.preview + // -- sets the working directory if( cmdRun.workDir ) config.workDir = cmdRun.workDir @@ -689,7 +697,7 @@ class ConfigBuilder { if( cmdRun.withTower != '-' ) config.tower.endpoint = cmdRun.withTower else if( !config.tower.endpoint ) - config.tower.endpoint = 'https://api.tower.nf' + config.tower.endpoint = 'https://api.cloud.seqera.io' } // -- set wave options @@ -841,10 +849,6 @@ class ConfigBuilder { } return result } - else if( config instanceof GString ) { - final holdSecrets = config.values.any { it instanceof SecretHolder } - return holdSecrets ? config : config.toString() - } else { return config } @@ -900,6 +904,7 @@ class ConfigBuilder { final config = new ConfigBuilder() .setShowClosures(true) + .setStripSecrets(true) .setOptions(cmdRun.launcher.options) .setCmdRun(cmdRun) .setBaseDir(baseDir) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy index e136316849..2780ef34e3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigClosurePlaceholder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigField.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigField.groovy index c729894c1d..a0af320039 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigField.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigField.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigMap.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigMap.groovy index eb9544a814..52f5de92f2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigMap.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigMap.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,6 @@ package nextflow.config import groovy.transform.CompileStatic -import nextflow.secret.SecretHolder -import nextflow.secret.SecretsProvider - /** * Represent Nextflow config as Map * @@ -39,42 +36,4 @@ class ConfigMap extends LinkedHashMap { super(content) } - @Override - Object get(Object key) { - final result = super.get(key) - // check if it's a secret value - if( result instanceof SecretHolder && result.isBound() ) { - return result.call() - } - return result - } - - ConfigMap withSecretProvider(SecretsProvider provider) { - withSecretProvider0(provider,this) - return this - } - - private withSecretProvider0(SecretsProvider provider, Map map) { - for( Object key : map.keySet() ) { - def entry = map.get(key) - // traverse nested config map objects - if( entry instanceof Map ) { - withSecretProvider0(provider, entry) - } - // look for all secret holders in the config map - // and bind the secrets provider - if( entry instanceof SecretHolder ) { - entry.bind(provider) - } - // same bind secret holders in Gstring objects - else if( entry instanceof GString ) { - final str = (GString)entry - for( Object value : str.getValues() ) { - if( value instanceof SecretHolder ) { - value.bind(provider) - } - } - } - } - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy index af05ba8687..787f1da57d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigParser.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -106,6 +106,8 @@ class ConfigParser { private boolean renderClosureAsString + private boolean stripSecrets + private Grengine grengine ConfigParser() { @@ -168,6 +170,8 @@ class ConfigParser { // set the required base script def config = new CompilerConfiguration() config.scriptBaseClass = ConfigBase.class.name + if( stripSecrets ) + config.addCompilationCustomizers(new ASTTransformationCustomizer(StripSecretsXform)) def params = [:] if( renderClosureAsString ) params.put('renderClosureAsString', true) @@ -186,6 +190,11 @@ class ConfigParser { return this } + ConfigParser setStripSecrets(boolean value) { + this.stripSecrets = value + return this + } + /** * Sets any additional variables that should be placed into the binding when evaluating Config scripts */ @@ -470,6 +479,7 @@ class ConfigParser { // disable include parsing when required script.setIgnoreIncludes(ignoreIncludes) script.setRenderClosureAsString(renderClosureAsString) + script.setStripSecrets(stripSecrets) // -- set the binding and run script.binding = binding diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransform.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransform.groovy index faf0ea9b5a..4dc8164aa4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransform.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransform.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy index 08c928115b..e6d8723388 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigTransformImpl.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package nextflow.config + import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.codehaus.groovy.ast.ASTNode @@ -203,5 +204,4 @@ class ConfigTransformImpl implements ASTTransformation { } } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/config/Manifest.groovy b/modules/nextflow/src/main/groovy/nextflow/config/Manifest.groovy index 9353561360..ad06fa0a49 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/Manifest.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/Manifest.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/config/StripSecretsXform.groovy b/modules/nextflow/src/main/groovy/nextflow/config/StripSecretsXform.groovy new file mode 100644 index 0000000000..c773944c0b --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/config/StripSecretsXform.groovy @@ -0,0 +1,37 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.config + +import java.lang.annotation.ElementType +import java.lang.annotation.Retention +import java.lang.annotation.RetentionPolicy +import java.lang.annotation.Target + +import org.codehaus.groovy.transform.GroovyASTTransformationClass + +/** + * AST transformation that replaces properties prefixed with `secrets.` + * with a static string + * + * @author Paolo Di Tommaso + */ +@Retention(RetentionPolicy.SOURCE) +@Target(ElementType.METHOD) +@GroovyASTTransformationClass(classes = [StripSecretsXformImpl]) +@interface StripSecretsXform { +} diff --git a/modules/nextflow/src/main/groovy/nextflow/config/StripSecretsXformImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/config/StripSecretsXformImpl.groovy new file mode 100644 index 0000000000..12cd6d7619 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/config/StripSecretsXformImpl.groovy @@ -0,0 +1,119 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.config + +import static org.codehaus.groovy.ast.tools.GeneralUtils.* + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.codehaus.groovy.ast.ASTNode +import org.codehaus.groovy.ast.ClassCodeExpressionTransformer +import org.codehaus.groovy.ast.ClassNode +import org.codehaus.groovy.ast.expr.ArgumentListExpression +import org.codehaus.groovy.ast.expr.ClosureExpression +import org.codehaus.groovy.ast.expr.ConstantExpression +import org.codehaus.groovy.ast.expr.Expression +import org.codehaus.groovy.ast.expr.MethodCallExpression +import org.codehaus.groovy.ast.expr.PropertyExpression +import org.codehaus.groovy.ast.expr.VariableExpression +import org.codehaus.groovy.control.CompilePhase +import org.codehaus.groovy.control.SourceUnit +import org.codehaus.groovy.transform.ASTTransformation +import org.codehaus.groovy.transform.GroovyASTTransformation +/** + * AST transformation that replaces properties prefixed with `secrets.` + * with a static string + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +@GroovyASTTransformation(phase = CompilePhase.CONVERSION) +class StripSecretsXformImpl implements ASTTransformation { + + private SourceUnit unit + + @Override + void visit(ASTNode[] nodes, SourceUnit source) { + this.unit = source + createVisitor().visitClass((ClassNode)nodes[1]) + } + + protected ClassCodeExpressionTransformer createVisitor() { + + new ClassCodeExpressionTransformer() { + + private boolean isIncludeConfigArgument + + protected SourceUnit getSourceUnit() { unit } + + @Override + Expression transform(Expression expr) { + if (expr == null) + return null + + final result = replaceProperty(expr) + if( result ) { + return result + } + if( expr instanceof MethodCallExpression ) { + return transformMethodCall(expr as MethodCallExpression) + } + if( expr instanceof ClosureExpression) { + visitClosureExpression(expr) + } + return super.transform(expr) + } + + protected Expression transformMethodCall(MethodCallExpression call) { + if( call.methodAsString=='includeConfig' && call.arguments instanceof ArgumentListExpression ) { + // flag method calls to `includeConfig` to preserve the use of secret properties + // for context see https://github.com/nextflow-io/nextflow/pull/4177 + isIncludeConfigArgument = true + try { + return super.transform(call) + } + finally { + isIncludeConfigArgument = false + } + } + else { + super.transform(call) + } + } + + protected Expression replaceProperty(Expression expr) { + if( expr !instanceof PropertyExpression ) + return null + final p = expr as PropertyExpression + final isSecretProperty = + p.objectExpression instanceof VariableExpression + && p.objectExpression.text=='secrets' + && p.property instanceof ConstantExpression + if( isSecretProperty && !isIncludeConfigArgument ) { + // replace the reference to a secrets property into a constant + // in order to disclose the secret value when using the `nextflow config` command + return constX("secrets.${p.propertyAsString}".toString()) + } + return null + } + + } + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ApptainerBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ApptainerBuilder.groovy index 3d95a8e82c..e381c9b163 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ApptainerBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ApptainerBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ApptainerCache.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ApptainerCache.groovy index 27c140409a..9837aeb328 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ApptainerCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ApptainerCache.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudBuilder.groovy index b72aa61569..68e2cd2ffe 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ */ package nextflow.container +import java.nio.file.Path +import java.nio.file.Paths import groovy.transform.CompileStatic import groovy.util.logging.Slf4j /** @@ -29,7 +31,11 @@ import groovy.util.logging.Slf4j @CompileStatic @Slf4j class CharliecloudBuilder extends ContainerBuilder { + + protected boolean useSquash + protected boolean writeFake + CharliecloudBuilder(String name) { this.image = name } @@ -46,6 +52,12 @@ class CharliecloudBuilder extends ContainerBuilder { if( params.containsKey('runOptions') ) addRunOptions(params.runOptions.toString()) + if ( params.containsKey('useSquash') ) + this.useSquash = params.useSquash?.toString() == 'true' + + if ( params.containsKey('writeFake') ) + this.writeFake = params.writeFake?.toString() == 'true' + if( params.containsKey('readOnlyInputs') ) this.readOnlyInputs = params.readOnlyInputs?.toString() == 'true' @@ -60,8 +72,37 @@ class CharliecloudBuilder extends ContainerBuilder { @Override CharliecloudBuilder build(StringBuilder result) { assert image + def imageStorage = Paths.get(image).parent.parent + def imageToRun = String + + if (!writeFake) { + // define image to run, if --write-fake is not used this is a copy of the image in the current workDir + imageToRun = '"$NXF_TASK_WORKDIR"/container_' + image.split('/')[-1] + + // optional squash + if (useSquash) { + imageToRun = imageToRun + '.squashfs' + } + + result << 'ch-convert -i ch-image --storage ' + // handle storage to deal with cases where CH_IMAGE_STORAGE is not set + result << imageStorage + result << ' ' + result << image.split('/')[-1] + result << ' ' + result << imageToRun + result << ' && ' + } result << 'ch-run --unset-env="*" -c "$NXF_TASK_WORKDIR" --set-env ' + + if (writeFake) { + result << '--write-fake ' + // if we are using writeFake we do not need to create a temporary imagae + // image is run by name from the storage directory + imageToRun = image.split('/')[-1] + } + if (!readOnlyInputs) result << '-w ' @@ -74,8 +115,8 @@ class CharliecloudBuilder extends ContainerBuilder { if( runOptions ) result << runOptions.join(' ') << ' ' - - result << image + + result << imageToRun result << ' --' runCommand = result.toString() diff --git a/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudCache.groovy b/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudCache.groovy index 7f0f074115..5ce98b9030 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/CharliecloudCache.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,6 +49,8 @@ class CharliecloudCache { private Duration pullTimeout = Duration.of('20min') + private String registry + /** Only for debugging purpose - do not use */ @PackageScope CharliecloudCache() {} @@ -77,8 +79,13 @@ class CharliecloudCache { String simpleName(String imageUrl) { def p = imageUrl.indexOf('://') def name = p != -1 ? imageUrl.substring(p+3) : imageUrl + + // add registry + if( registry ) + name = registry + name + name = name.replace(':','+').replace('/','%') - return name + return name } /** @@ -168,20 +175,21 @@ class CharliecloudCache { return localPath } - // final file = new File("${localPath.parent.parent.parent}/.${localPath.name}.lock") - final file = new File("${localPath.parent.parent.parent}/.ch-pulling.lock") - final wait = "Another Nextflow instance is pulling the image $imageUrl with Charliecloud -- please wait until the download completes" - final err = "Unable to acquire exclusive lock after $pullTimeout on file: $file" - - final mutex = new FileMutex(target: file, timeout: pullTimeout, waitMessage: wait, errorMessage: err) - try { - mutex .lock { downloadCharliecloudImage0(imageUrl, localPath) } - } - finally { - file.delete() - } - + int count = 0; + int maxTries = 5; + boolean imagePulled = false + while(!imagePulled) { + try { + downloadCharliecloudImage0(imageUrl, localPath) + imagePulled = true + } catch (e) { + if (++count == maxTries) throw e + log.info "Another image is currently pulled. Attempting again in 30 seconds [$count/$maxTries]" + Thread.sleep(30000) + } + } return localPath + } @@ -231,7 +239,7 @@ class CharliecloudCache { def status = proc.exitValue() if( status != 0 ) { consumer.join() - def msg = "Charliecloud failed to pull image\n command: $cmd\n status : $status\n message:\n" + def msg = "Charliecloud failed to pull image\n command: $cmd\n status : $status\n hint : Try and increase charliecloud.pullTimeout in the config (current is \"${pullTimeout}\")\n message:\n" msg += err.toString().trim().indent(' ') throw new IllegalStateException(msg) } diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy index e552a8aaaa..dda03dfdbb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy index 44cc070156..0709832455 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy index ab29fddb72..4eee91581f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ContainerNameValidator.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ContainerNameValidator.groovy index ccbdb2de3f..d5fc42b272 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ContainerNameValidator.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ContainerNameValidator.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/DockerBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/DockerBuilder.groovy index 08c8bbe182..0a648ae418 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/DockerBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/DockerBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/PodmanBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/PodmanBuilder.groovy index 84ff98a72f..d111547128 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/PodmanBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/PodmanBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/ShifterBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/ShifterBuilder.groovy index d1cb33bfc1..3dbfe01707 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/ShifterBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/ShifterBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy index 9a02b85765..78394c06f7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/SingularityBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy b/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy index 2fa7c85db4..19277a163c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/SingularityCache.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,9 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.LazyDataflowVariable +import nextflow.Const import nextflow.Global +import nextflow.SysEnv import nextflow.file.FileMutex import nextflow.util.Duration import nextflow.util.Escape @@ -68,7 +70,7 @@ class SingularityCache { */ SingularityCache(ContainerConfig config, Map env=null) { this.config = config - this.env = env ?: System.getenv() + this.env = env ?: SysEnv.get() } /** @@ -155,7 +157,7 @@ class SingularityCache { def workDir = Global.session.workDir if( workDir.fileSystem != FileSystems.default ) { // when the work dir is a remote path use the local launch directory to cache image files - workDir = Path.of('.nextflow').toAbsolutePath() + workDir = Const.appCacheDir.toAbsolutePath() } missingCacheDir = true @@ -301,7 +303,7 @@ class SingularityCache { def status = proc.exitValue() if( status != 0 ) { consumer.join() - def msg = "Failed to pull singularity image\n command: $cmd\n status : $status\n message:\n" + def msg = "Failed to pull singularity image\n command: $cmd\n status : $status\n hint : Try and increase ${binaryName}.pullTimeout in the config (current is \"${pullTimeout}\")\n message:\n" msg += err.toString().trim().indent(' ') throw new IllegalStateException(msg) } diff --git a/modules/nextflow/src/main/groovy/nextflow/container/UdockerBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/container/UdockerBuilder.groovy index 31bd44332a..919d023ad9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/UdockerBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/UdockerBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainerInspectMode.groovy b/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainerInspectMode.groovy index 593736c62f..4351b5ffea 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainerInspectMode.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainerInspectMode.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainersInspector.groovy b/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainersInspector.groovy index b4ee859969..1a2e6827f3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainersInspector.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/inspect/ContainersInspector.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerInfo.groovy b/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerInfo.groovy index 66d21abcb9..905c6ae4f7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerInfo.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerInfo.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolver.groovy index dd534d5db3..4ccd19c2a1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolver.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolverProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolverProvider.groovy index 080fbb35c7..26ecae1b9b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolverProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/resolver/ContainerResolverProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy b/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy index 34cafe65b5..8e83ccbd32 100644 --- a/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/container/resolver/DefaultContainerResolver.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/daemon/DaemonLauncher.groovy b/modules/nextflow/src/main/groovy/nextflow/daemon/DaemonLauncher.groovy index 6533e9e94c..93c86bd1d3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/daemon/DaemonLauncher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/daemon/DaemonLauncher.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy index e61dac5985..0289f948a8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DagRenderer.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DagRenderer.groovy index 0d3bfcd252..145bc96db4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/DagRenderer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/DagRenderer.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DotRenderer.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DotRenderer.groovy index f9b5a36e41..e42bd2cf5a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/DotRenderer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/DotRenderer.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/GraphVizRenderer.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/GraphVizRenderer.groovy index 3106890b9c..6490bc7b16 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/GraphVizRenderer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/GraphVizRenderer.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/MermaidHtmlRenderer.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/MermaidHtmlRenderer.groovy index f2d175b89b..eb99b44e71 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/MermaidHtmlRenderer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/MermaidHtmlRenderer.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/MermaidRenderer.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/MermaidRenderer.groovy index 5538dfed8a..41d9f39068 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/MermaidRenderer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/MermaidRenderer.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/MultipleInputChannelException.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/MultipleInputChannelException.groovy index eb9f98af81..1f34dd972d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/MultipleInputChannelException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/MultipleInputChannelException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/MultipleOutputChannelException.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/MultipleOutputChannelException.groovy index de7753bcf8..bbd2a9de34 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/MultipleOutputChannelException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/MultipleOutputChannelException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy index 68d56a12d5..c7885ee71a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/datasource/SraExplorer.groovy b/modules/nextflow/src/main/groovy/nextflow/datasource/SraExplorer.groovy index f4d1aa6649..8c0f118beb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/datasource/SraExplorer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/datasource/SraExplorer.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/AbortRunException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/AbortRunException.groovy index bcbad82370..4a61e30269 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/AbortRunException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/AbortRunException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/AbortSignalException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/AbortSignalException.groovy index 164962feb2..3602c11b2b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/AbortSignalException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/AbortSignalException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/AmbiguousPipelineNameException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/AmbiguousPipelineNameException.groovy index f78f42f871..df7b5e3a79 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/AmbiguousPipelineNameException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/AmbiguousPipelineNameException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ConfigParseException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ConfigParseException.groovy index ca634be462..8d5e1071d6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ConfigParseException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ConfigParseException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateChannelNameException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateChannelNameException.groovy index b4b17c4777..21ba50f98a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateChannelNameException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateChannelNameException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateModuleFunctionException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateModuleFunctionException.groovy index 017802abe6..ccbf9b17b4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateModuleFunctionException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateModuleFunctionException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateProcessInvocation.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateProcessInvocation.groovy index d41641f91d..99e2c486d1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateProcessInvocation.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateProcessInvocation.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/FailedGuardException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/FailedGuardException.groovy index 1d436b3cf6..5013a3d969 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/FailedGuardException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/FailedGuardException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalArityException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalArityException.groovy index f11f8f3cb2..010b439c7d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalArityException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalArityException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalConfigException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalConfigException.groovy index 7f6d54d147..6740fb9297 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalConfigException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalConfigException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalDirectiveException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalDirectiveException.groovy index 83bd819a80..56c95dd0c0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalDirectiveException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalDirectiveException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalFileException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalFileException.groovy index 17a01d409f..b8c20d1284 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalFileException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalFileException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalModulePath.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalModulePath.groovy index 8d4fd86de7..3b876df849 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalModulePath.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalModulePath.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/MissingFileException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/MissingFileException.groovy index 4423927f6e..95f51531a9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/MissingFileException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/MissingFileException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/MissingLibraryException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/MissingLibraryException.groovy index 9153ec912d..c819b82c64 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/MissingLibraryException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/MissingLibraryException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/MissingModuleComponentException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/MissingModuleComponentException.groovy index 87775587d9..2371e449c3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/MissingModuleComponentException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/MissingModuleComponentException.groovy @@ -19,7 +19,7 @@ class MissingModuleComponentException extends ProcessException { @PackageScope static String message(ScriptMeta meta, String name) { def result = "Cannot find a component with name '$name' in module: $meta.scriptPath" - def names = meta.getDefinitions().collect { it.name } + def names = meta.getDefinitions().findAll { it.name }.collect { it.name } def matches = names.closest(name) if( matches ) result += "\n\nDid you mean any of these?\n" + matches.collect { " $it"}.join('\n') + '\n' diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/MissingValueException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/MissingValueException.groovy index 8b0e7e2073..86b87e4811 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/MissingValueException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/MissingValueException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy index 02aabb9fff..45ad9e67ae 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/NodeTerminationException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/secret/SecretsContext.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy similarity index 59% rename from modules/nextflow/src/main/groovy/nextflow/secret/SecretsContext.groovy rename to modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy index 8c37f534a7..a2babd2b2e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/secret/SecretsContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2021, Sage-Bionetworks + * Copyright 2013-2023, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,28 +15,26 @@ * */ -package nextflow.secret +package nextflow.exception import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j /** - * Model a context to access secret values in nextflow config files - * + * Exception thrown when a command output returns a non-zero exit status + * * @author Paolo Di Tommaso */ -@Slf4j @CompileStatic -class SecretsContext { +class ProcessEvalException extends RuntimeException implements ShowOnlyExceptionMessage { - SecretsContext() {} + String command + String output + int status - @Override - Object getProperty(String name) { - if( metaClass.hasProperty(name) ) - return metaClass.getProperty(this,name) - else { - return new SecretHolder(name) - } + ProcessEvalException(String message, String command, String output, int status) { + super(message) + this.command = command + this.output = output + this.status = status } } diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessException.groovy index efdbe29149..7c762cc870 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessFailedException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessFailedException.groovy index add2af0962..9486dbfda0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessFailedException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessFailedException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessNonZeroExitStatusException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessNonZeroExitStatusException.groovy index 3e6f342072..665b7425c5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessNonZeroExitStatusException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessNonZeroExitStatusException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessStageException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessStageException.groovy index e07967f088..6e23844e61 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessStageException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessStageException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessSubmitException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessSubmitException.groovy index 4f9038fa30..68dfcff960 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessSubmitException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessSubmitException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessTemplateException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessTemplateException.groovy index 50c59f8857..1507306bb4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessTemplateException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessTemplateException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessUnrecoverableException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessUnrecoverableException.groovy index 932a7f00e0..567a63823e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ProcessUnrecoverableException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ProcessUnrecoverableException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/RateLimitExceededException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/RateLimitExceededException.groovy index 2747f964eb..4b53ac8630 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/RateLimitExceededException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/RateLimitExceededException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ScriptCompilationException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ScriptCompilationException.groovy index b842d4f1b7..e2642b0fc5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ScriptCompilationException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ScriptCompilationException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ScriptRuntimeException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ScriptRuntimeException.groovy index cb09625ba6..82766fa2d7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ScriptRuntimeException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ScriptRuntimeException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/ShowOnlyExceptionMessage.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/ShowOnlyExceptionMessage.groovy index 1b899ee453..b465b7a313 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/ShowOnlyExceptionMessage.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/ShowOnlyExceptionMessage.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/StopSplitIterationException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/StopSplitIterationException.groovy index f15ab0d82d..69d28940cf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/exception/StopSplitIterationException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/exception/StopSplitIterationException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy index cfad0b5246..6d6b7cc0a4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.nio.file.Path import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j +import nextflow.processor.TaskConfig import nextflow.processor.TaskMonitor import nextflow.processor.TaskPollingMonitor import nextflow.processor.TaskProcessor @@ -135,6 +136,23 @@ abstract class AbstractGridExecutor extends Executor { */ abstract protected List getDirectives(TaskRun task, List initial) + protected void addClusterOptionsDirective(TaskConfig config, List result) { + final opts = config.getClusterOptions() + if( opts instanceof Collection ) { + for( String it : opts ) { + result.add(it) + result.add('') + } + } + else if( opts instanceof CharSequence ) { + result.add(opts.toString()) + result.add('') + } + else if( opts != null ) { + throw new IllegalArgumentException("Unexpected value for clusterOptions process directive - offending value: $opts") + } + } + /** * Given a task returns a *clean* name used to submit the job to the grid engine. * That string must not contain blank or special shell characters e.g. parenthesis, etc diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashFunLib.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashFunLib.groovy index d506b3931a..7c9667f6ff 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashFunLib.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashFunLib.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashTemplateEngine.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashTemplateEngine.groovy index cd7a479b60..0dfe016e3d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashTemplateEngine.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashTemplateEngine.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy index 613c3dd706..9cb514a811 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package nextflow.executor +import static java.nio.file.StandardOpenOption.* + import java.nio.file.FileSystemException import java.nio.file.FileSystems import java.nio.file.Files @@ -29,17 +31,14 @@ import nextflow.container.ContainerBuilder import nextflow.container.DockerBuilder import nextflow.container.SingularityBuilder import nextflow.exception.ProcessException +import nextflow.extension.FilesEx import nextflow.file.FileHelper import nextflow.processor.TaskBean import nextflow.processor.TaskProcessor import nextflow.processor.TaskRun import nextflow.secret.SecretsLoader import nextflow.util.Escape - -import static java.nio.file.StandardOpenOption.* - import nextflow.util.MemoryUnit - /** * Builder to create the Bash script which is used to * wrap and launch the user task @@ -177,20 +176,76 @@ class BashWrapperBuilder { } } - protected String getOutputEnvCaptureSnippet(List names) { - def result = new StringBuilder() - result.append('\n') - result.append('# capture process environment\n') - result.append('set +u\n') - result.append('cd "$NXF_TASK_WORKDIR"\n') - for( int i=0; i ' : '>> ' ) - result.append(TaskRun.CMD_ENV) - result.append('\n') + /** + * Generate a Bash script to be appended to the task command script + * that takes care of capturing the process output environment variables + * and evaluation commands + * + * @param outEnvs + * The list of environment variables names whose value need to be captured + * @param outEvals + * The set of commands to be evaluated to determine the output value to be captured + * @return + * The Bash script to capture the output environment and eval commands + */ + protected String getOutputEnvCaptureSnippet(List outEnvs, Map outEvals) { + // load the env template + final template = BashWrapperBuilder.class + .getResourceAsStream('command-env.txt') + .newReader() + final binding = Map.of('env_file', TaskRun.CMD_ENV) + final result = new StringBuilder() + result.append( engine.render(template, binding) ) + appendOutEnv(result, outEnvs) + appendOutEval(result, outEvals) + return result.toString() + } + + /** + * Render a Bash script to capture the one or more env variables + * + * @param result A {@link StringBuilder} instance to which append the result Bash script + * @param outEnvs The environment variables to be captured + */ + protected void appendOutEnv(StringBuilder result, List outEnvs) { + if( outEnvs==null ) + outEnvs = List.of() + // out env + for( String key : outEnvs ) { + result << "#\n" + result << "echo $key=\"\${$key[@]}\" >> ${TaskRun.CMD_ENV}\n" + result << "echo /$key/ >> ${TaskRun.CMD_ENV}\n" + } + } + + /** + * Render a Bash script to capture the result of one or more commands + * evaluated in the task script context + * + * @param result + * A {@link StringBuilder} instance to which append the result Bash script + * @param outEvals + * A {@link Map} of key-value pairs modeling the commands to be evaluated; + * where the key represents the environment variable (name) holding the + * resulting output, and the pair value represent the Bash command to be + * evaluated. + */ + protected void appendOutEval(StringBuilder result, Map outEvals) { + if( outEvals==null ) + outEvals = Map.of() + // out eval + for( Map.Entry eval : outEvals ) { + result << "#\n" + result <<"nxf_eval_cmd STDOUT STDERR bash -c \"${eval.value.replace('"','\\\"')}\"\n" + result << 'status=$?\n' + result << 'if [ $status -eq 0 ]; then\n' + result << " echo $eval.key=\"\$STDOUT\" >> ${TaskRun.CMD_ENV}\n" + result << " echo /$eval.key/=exit:0 >> ${TaskRun.CMD_ENV}\n" + result << 'else\n' + result << " echo $eval.key=\"\$STDERR\" >> ${TaskRun.CMD_ENV}\n" + result << " echo /$eval.key/=exit:\$status >> ${TaskRun.CMD_ENV}\n" + result << 'fi\n' } - result.toString() } protected String stageCommand(String stagingScript) { @@ -239,13 +294,21 @@ class BashWrapperBuilder { */ final interpreter = TaskProcessor.fetchInterpreter(script) - if( outputEnvNames ) { - if( !isBash(interpreter) ) throw new IllegalArgumentException("Process output of type env is only allowed with Bash process command -- Current interpreter: $interpreter") - script += getOutputEnvCaptureSnippet(outputEnvNames) + /* + * append to the command script a prolog to capture the declared + * output environment (variable) and evaluation commands + */ + if( outputEnvNames || outputEvals ) { + if( !isBash(interpreter) && outputEnvNames ) + throw new IllegalArgumentException("Process output of type 'env' is only allowed with Bash process scripts -- Current interpreter: $interpreter") + if( !isBash(interpreter) && outputEvals ) + throw new IllegalArgumentException("Process output of type 'eval' is only allowed with Bash process scripts -- Current interpreter: $interpreter") + script += getOutputEnvCaptureSnippet(outputEnvNames, outputEvals) } final binding = new HashMap(20) binding.header_script = headerScript + binding.task_metadata = getTaskMetadata() binding.task_name = name binding.helpers_script = getHelpersScript() @@ -400,6 +463,31 @@ class BashWrapperBuilder { } } + protected String getTaskMetadata() { + final lines = new StringBuilder() + lines << '### ---\n' + lines << "### name: '${bean.name}'\n" + if( bean.arrayIndexName ) { + lines << '### array:\n' + lines << "### index-name: ${bean.arrayIndexName}\n" + lines << "### index-start: ${bean.arrayIndexStart}\n" + lines << "### work-dirs:\n" + for( Path it : bean.arrayWorkDirs ) + lines << "### - ${Escape.path(FilesEx.toUriString(it))}\n" + } + + if( containerConfig?.isEnabled() ) + lines << "### container: '${bean.containerImage}'\n" + + if( outputFiles.size() > 0 ) { + lines << '### outputs:\n' + for( final output : bean.outputFiles ) + lines << "### - '${output}'\n" + } + + lines << '### ...\n' + } + protected String getHelpersScript() { def result = new StringBuilder() diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BatchCleanup.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BatchCleanup.groovy index 55ba53199c..fa5ccd0931 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BatchCleanup.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BatchCleanup.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BridgeExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BridgeExecutor.groovy index 8284ecd5c0..0c086ee124 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BridgeExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BridgeExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * Copyright 2022, CEA-CNRGH * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -76,9 +76,7 @@ class BridgeExecutor extends AbstractGridExecutor { } // other cluster options - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' - } + addClusterOptionsDirective(task.config, result) return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/CachedTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/CachedTaskHandler.groovy index bfec23dcd9..841db3d876 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/CachedTaskHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/CachedTaskHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/CondorExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/CondorExecutor.groovy index 0bcf5ffa66..6e7dcafd88 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/CondorExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/CondorExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -75,10 +75,10 @@ class CondorExecutor extends AbstractGridExecutor { result << "periodic_remove = (RemoteWallClockTime - CumulativeSuspensionTime) > ${task.config.getTime().toSeconds()}".toString() } - if( task.config.clusterOptions ) { - def opts = task.config.clusterOptions + if( task.config.getClusterOptions() ) { + def opts = task.config.getClusterOptions() if( opts instanceof Collection ) { - result.addAll(opts) + result.addAll(opts as Collection) } else { result.addAll( opts.toString().tokenize(';\n').collect{ it.trim() }) diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy index 0b12dd9c8a..53b2ca92b1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/CrgExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package nextflow.executor import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.processor.TaskArrayRun import nextflow.processor.TaskRun /** * An executor specialised for CRG cluster @@ -41,9 +42,18 @@ class CrgExecutor extends SgeExecutor { task.config.penv = 'smp' } + if( task instanceof TaskArrayRun ) { + final arraySize = task.getArraySize() + result << '-t' << "1-${arraySize}".toString() + } + result << '-N' << getJobNameFor(task) - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'y' + + if( task !instanceof TaskArrayRun ) { + result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) + result << '-j' << 'y' + } + result << '-terse' << '' // note: directive need to be returned as pairs /* @@ -95,9 +105,7 @@ class CrgExecutor extends SgeExecutor { } // -- at the end append the command script wrapped file name - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' - } + addClusterOptionsDirective(task.config, result) return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy index 18742637ba..1d6d92f37e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,7 +81,7 @@ abstract class Executor { * * @param task A {@code TaskRun} instance */ - final void submit( TaskRun task ) { + void submit( TaskRun task ) { log.trace "Scheduling process: ${task}" if( session.isTerminated() ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/FluxExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/FluxExecutor.groovy index 1b64dba81a..25d96fc78e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/FluxExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/FluxExecutor.groovy @@ -20,6 +20,7 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.processor.TaskConfig import nextflow.processor.TaskRun /** * Processor for Flux Framework executor @@ -90,18 +91,25 @@ class FluxExecutor extends AbstractGridExecutor { } // Any extra cluster options the user wants! - // Options tokenized with ; akin to OarExecutor - if( task.config.clusterOptions ) { + addClusterOptionsDirective(task.config, result) + result << '/bin/bash' << scriptFile.getName() + return result + } + + @Override + protected void addClusterOptionsDirective(TaskConfig config, List result) { + final opts = config.getClusterOptions() + final str = opts instanceof Collection ? opts.join(' ') : opts?.toString() + + if( str ) { // Split by space - for (String item : task.config.clusterOptions.toString().tokenize(' ')) { + for (String item : str.tokenize(' ')) { if ( item ) { result << item.stripIndent(true).trim() } } } - result << '/bin/bash' << scriptFile.getName() - return result } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy index 3ad6430856..11d310b600 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ import nextflow.exception.ProcessNonZeroExitStatusException import nextflow.file.FileHelper import nextflow.fusion.FusionAwareTask import nextflow.fusion.FusionHelper +import nextflow.processor.TaskArrayRun import nextflow.processor.TaskHandler import nextflow.processor.TaskRun import nextflow.trace.TraceRecord @@ -100,6 +101,12 @@ class GridTaskHandler extends TaskHandler implements FusionAwareTask { this.sanityCheckInterval = duration } + @Override + void prepareLauncher() { + // -- create the wrapper script + createTaskWrapper(task).build() + } + protected ProcessBuilder createProcessBuilder() { // -- log the qsub command @@ -254,17 +261,15 @@ class GridTaskHandler extends TaskHandler implements FusionAwareTask { void submit() { ProcessBuilder builder = null try { - // -- create the wrapper script - createTaskWrapper(task).build() // -- start the execution and notify the event to the monitor builder = createProcessBuilder() // -- forward the job launcher script to the command stdin if required final stdinScript = executor.pipeLauncherScript() ? stdinLauncherScript() : null // -- execute with a re-triable strategy final result = safeExecute( () -> processStart(builder, stdinScript) ) - // -- save the JobId in the - this.jobId = executor.parseJobId(result) - this.status = SUBMITTED + // -- save the job id + final jobId = (String)executor.parseJobId(result) + updateStatus(jobId) log.debug "[${executor.name.toUpperCase()}] submitted process ${task.name} > jobId: $jobId; workDir: ${task.workDir}" } @@ -281,9 +286,21 @@ class GridTaskHandler extends TaskHandler implements FusionAwareTask { status = COMPLETED throw new ProcessFailedException("Error submitting process '${task.name}' for execution", e ) } - } + private void updateStatus(String jobId) { + if( task instanceof TaskArrayRun ) { + for( int i=0; i getDirectives(TaskRun task, List result) { - result << '-o' << task.workDir.resolve(TaskRun.CMD_LOG).toString() + if( task !instanceof TaskArrayRun ) { + result << '-o' << task.workDir.resolve(TaskRun.CMD_LOG).toString() + } // add other parameters (if any) if( task.config.queue ) { @@ -104,14 +108,49 @@ class LsfExecutor extends AbstractGridExecutor { } // -- the job name - result << '-J' << getJobNameFor(task) + if( task instanceof TaskArrayRun ) { + final arraySize = task.getArraySize() + result << '-J' << "${getJobNameFor(task)}[1-${arraySize}]".toString() + } + else { + result << '-J' << getJobNameFor(task) + } // -- at the end append the command script wrapped file name - result.addAll( task.config.getClusterOptionsAsList() ) + addClusterOptionsDirective(task.config, result) + + // add account from config + final account = session.getExecConfigProp(getName(), 'account', null) as String + if( account ) { + result << '-G' << account + } return result } + @Override + protected void addClusterOptionsDirective(TaskConfig config, List result) { + final opts = config.getClusterOptions() + // when cluster options are defined as a list rely on default behavior + if( opts instanceof Collection ) { + super.addClusterOptionsDirective(config,result) + } + // when cluster options are a string value use the `getClusterOptionsAsList` for backward compatibility + else if( opts instanceof CharSequence ) { + result.addAll( config.getClusterOptionsAsList() ) + } + else if( opts != null ) { + throw new IllegalArgumentException("Unexpected value for clusterOptions process directive - offending value: $opts") + } + } + + @Override + String sanitizeJobName( String name ) { + // LSF does not allow square brackets in job names except for job arrays + name = name.replace('[','').replace(']','') + // Old LSF versions do not allow job names longer than 511 chars + name.size()>511 ? name.substring(0,511) : name + } /** * The command line to submit this job @@ -304,4 +343,21 @@ class LsfExecutor extends AbstractGridExecutor { boolean isFusionEnabled() { return FusionHelper.isFusionEnabled(session) } + + @Override + String getArrayIndexName() { + return 'LSB_JOBINDEX' + } + + @Override + int getArrayIndexStart() { + return 1 + } + + @Override + String getArrayTaskId(String jobId, int index) { + assert jobId, "Missing 'jobId' argument" + return "${jobId}[${index + 1}]" + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/MoabExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/MoabExecutor.groovy index 32eb953d8c..1271a07a41 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/MoabExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/MoabExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -73,9 +73,7 @@ class MoabExecutor extends AbstractGridExecutor { } // -- at the end append the command script wrapped file name - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' - } + addClusterOptionsDirective(task.config, result) return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/NopeExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/NopeExecutor.groovy index 798c068275..ce7dcf4ad4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/NopeExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/NopeExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/NqsiiExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/NqsiiExecutor.groovy index 33319dce8b..7cc887d7f7 100755 --- a/modules/nextflow/src/main/groovy/nextflow/executor/NqsiiExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/NqsiiExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,9 +71,7 @@ class NqsiiExecutor extends AbstractGridExecutor { } // -- at the end append the command script wrapped file name - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' - } + addClusterOptionsDirective(task.config, result) return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/OarExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/OarExecutor.groovy index 7c8cbb77e7..f737b6b9f1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/OarExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/OarExecutor.groovy @@ -20,6 +20,7 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.processor.TaskConfig import nextflow.processor.TaskRun /** * Processor for OAR resource manager @@ -80,16 +81,27 @@ class OarExecutor extends AbstractGridExecutor { } // -- at the end append the command script wrapped file name - // Options need to be semicolon ";" separated, if several are needed - if( task.config.clusterOptions ) { - for (String item : task.config.clusterOptions.toString().tokenize(';')) { - result << item << '' - } - } + addClusterOptionsDirective(task.config, result) return result } + @Override + void addClusterOptionsDirective(TaskConfig config, List result) { + final opts = config.getClusterOptions() + if( opts instanceof Collection ) { + for( String it in opts ) { + result << it << '' + } + } + // Options need to be semicolon ";" separated, if several are needed + else if( opts instanceof CharSequence ) { + for (String item : opts.toString().tokenize(';')) { + result << item << '' + } + } + } + String getHeaderToken() { '#OAR' } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy index 40d749127d..6f7bc7d9b4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.processor.TaskArrayRun import nextflow.processor.TaskRun /** * Implements a executor for PBS/Torque cluster @@ -29,7 +30,7 @@ import nextflow.processor.TaskRun */ @Slf4j @CompileStatic -class PbsExecutor extends AbstractGridExecutor { +class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor { private static Pattern OPTS_REGEX = ~/(?:^|\s)-l.+/ @@ -43,9 +44,17 @@ class PbsExecutor extends AbstractGridExecutor { protected List getDirectives( TaskRun task, List result ) { assert result !=null + if( task instanceof TaskArrayRun ) { + final arraySize = task.getArraySize() + result << '-J' << "0-${arraySize - 1}".toString() + } + result << '-N' << getJobNameFor(task) - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'oe' + + if( task !instanceof TaskArrayRun ) { + result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) + result << '-j' << 'oe' + } // the requested queue name if( task.config.queue ) { @@ -54,7 +63,7 @@ class PbsExecutor extends AbstractGridExecutor { // task cpus if( task.config.getCpus() > 1 ) { - if( matchOptions(task.config.clusterOptions?.toString()) ) { + if( matchOptions(task.config.getClusterOptionsAsString()) ) { log.warn1 'cpus directive is ignored when clusterOptions contains -l option\ntip: clusterOptions = { "-l nodes=1:ppn=${task.cpus}:..." }' } else { @@ -74,11 +83,15 @@ class PbsExecutor extends AbstractGridExecutor { result << "-l" << "mem=${task.config.getMemory().toString().replaceAll(/[\s]/,'').toLowerCase()}".toString() } - // -- at the end append the command script wrapped file name - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' + // add account from config + final account = session.getExecConfigProp(getName(), 'account', null) as String + if( account ) { + result << '-P' << account } + // -- at the end append the command script wrapped file name + addClusterOptionsDirective(task.config, result) + return result } @@ -174,4 +187,21 @@ class PbsExecutor extends AbstractGridExecutor { static protected boolean matchOptions(String value) { value ? OPTS_REGEX.matcher(value).find() : null } + + @Override + String getArrayIndexName() { + return 'PBS_ARRAY_INDEX' + } + + @Override + int getArrayIndexStart() { + return 0 + } + + @Override + String getArrayTaskId(String jobId, int index) { + assert jobId, "Missing 'jobId' argument" + return jobId.replace('[]', "[$index]") + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy index 0aa33048de..1672ca200d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package nextflow.executor import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.processor.TaskArrayRun import nextflow.processor.TaskRun /** * Implements a executor for PBSPro cluster executor @@ -44,17 +45,23 @@ class PbsProExecutor extends PbsExecutor { @Override protected List getDirectives(TaskRun task, List result ) { assert result !=null - + + if( task instanceof TaskArrayRun ) { + final arraySize = task.getArraySize() + result << '-J' << "0-${arraySize - 1}".toString() + } + // when multiple competing directives are provided, only the first one will take effect // therefore clusterOptions is added as first to give priority over other options as expected // by the clusterOptions semantics -- see https://github.com/nextflow-io/nextflow/pull/2036 - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' - } + addClusterOptionsDirective(task.config, result) result << '-N' << getJobNameFor(task) - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'oe' + + if( task !instanceof TaskArrayRun ) { + result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) + result << '-j' << 'oe' + } // the requested queue name if( task.config.queue ) { @@ -70,7 +77,7 @@ class PbsProExecutor extends PbsExecutor { res << "mem=${task.config.getMemory().getMega()}mb".toString() } if( res ) { - if( matchOptions(task.config.clusterOptions?.toString()) ) { + if( matchOptions(task.config.getClusterOptionsAsString()) ) { log.warn1 'cpus and memory directives are ignored when clusterOptions contains -l option\ntip: clusterOptions = { "-l select=1:ncpus=${task.cpus}:mem=${task.memory.toMega()}mb:..." }' } else { @@ -84,6 +91,12 @@ class PbsProExecutor extends PbsExecutor { result << "-l" << "walltime=${duration.format('HH:mm:ss')}".toString() } + // add account from config + final account = session.getExecConfigProp(getName(), 'account', null) as String + if( account ) { + result << '-P' << account + } + return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/ScriptFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/ScriptFileCopyStrategy.groovy index 84f67c4683..6aba0419da 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/ScriptFileCopyStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/ScriptFileCopyStrategy.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy index 52ebab4886..1f1c2a6304 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SgeExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import java.nio.file.Path import groovy.transform.CompileStatic import nextflow.fusion.FusionHelper +import nextflow.processor.TaskArrayRun import nextflow.processor.TaskRun /** * Execute a task script by running it on the SGE/OGE cluster @@ -26,7 +27,7 @@ import nextflow.processor.TaskRun * @author Paolo Di Tommaso */ @CompileStatic -class SgeExecutor extends AbstractGridExecutor { +class SgeExecutor extends AbstractGridExecutor implements TaskArrayExecutor { /** * Gets the directives to submit the specified task to the cluster for execution @@ -37,9 +38,18 @@ class SgeExecutor extends AbstractGridExecutor { */ protected List getDirectives(TaskRun task, List result) { + if( task instanceof TaskArrayRun ) { + final arraySize = task.getArraySize() + result << '-t' << "1-${arraySize}".toString() + } + result << '-N' << getJobNameFor(task) - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) - result << '-j' << 'y' + + if( task !instanceof TaskArrayRun ) { + result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) + result << '-j' << 'y' + } + result << '-terse' << '' // note: directive need to be returned as pairs /* @@ -74,9 +84,7 @@ class SgeExecutor extends AbstractGridExecutor { } // -- at the end append the command script wrapped file name - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' - } + addClusterOptionsDirective(task.config, result) return result } @@ -114,8 +122,14 @@ class SgeExecutor extends AbstractGridExecutor { if( entry.toString().isLong() ) return entry + if( (id=entry.tokenize('.').get(0)).isLong() ) + return id + if( entry.startsWith('Your job') && entry.endsWith('has been submitted') && (id=entry.tokenize().get(2)) ) return id + + if( entry.startsWith('Your job array') && entry.endsWith('has been submitted') && (id=entry.tokenize().get(3)) ) + return id.tokenize('.').get(0) } throw new IllegalStateException("Invalid SGE submit response:\n$text\n\n") @@ -185,4 +199,20 @@ class SgeExecutor extends AbstractGridExecutor { boolean isFusionEnabled() { return FusionHelper.isFusionEnabled(session) } + + @Override + String getArrayIndexName() { + return 'SGE_TASK_ID' + } + + @Override + int getArrayIndexStart() { + return 1 + } + + @Override + String getArrayTaskId(String jobId, int index) { + return "${jobId}.${index}" + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy index b8d7564171..e1a92b172c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy index f195e280e6..8210f24c8b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SlurmExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.fusion.FusionHelper +import nextflow.processor.TaskArrayRun +import nextflow.processor.TaskConfig import nextflow.processor.TaskRun /** * Processor for SLURM resource manager @@ -33,18 +35,17 @@ import nextflow.processor.TaskRun */ @Slf4j @CompileStatic -class SlurmExecutor extends AbstractGridExecutor { +class SlurmExecutor extends AbstractGridExecutor implements TaskArrayExecutor { static private Pattern SUBMIT_REGEX = ~/Submitted batch job (\d+)/ private boolean perCpuMemAllocation - private boolean hasSignalOpt(Map config) { - def opts = config.clusterOptions?.toString() + private boolean hasSignalOpt(TaskConfig config) { + final opts = config.getClusterOptionsAsString() return opts ? opts.contains('--signal ') || opts.contains('--signal=') : false } - /** * Gets the directives to submit the specified task to the cluster for execution * @@ -54,8 +55,18 @@ class SlurmExecutor extends AbstractGridExecutor { */ protected List getDirectives(TaskRun task, List result) { + if( task instanceof TaskArrayRun ) { + final arraySize = task.getArraySize() + result << '--array' << "0-${arraySize - 1}".toString() + } + result << '-J' << getJobNameFor(task) - result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) // -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE + + if( task !instanceof TaskArrayRun ) { + // -o OUTFILE and no -e option => stdout and stderr merged to stdout/OUTFILE + result << '-o' << quote(task.workDir.resolve(TaskRun.CMD_LOG)) + } + result << '--no-requeue' << '' // note: directive need to be returned as pairs if( !hasSignalOpt(task.config) ) { @@ -91,8 +102,12 @@ class SlurmExecutor extends AbstractGridExecutor { } // -- at the end append the command script wrapped file name - if( task.config.clusterOptions ) { - result << task.config.clusterOptions.toString() << '' + addClusterOptionsDirective(task.config, result) + + // add slurm account from config + final account = session.getExecConfigProp(getName(), 'account', null) as String + if( account ) { + result << '-A' << account } return result @@ -211,4 +226,21 @@ class SlurmExecutor extends AbstractGridExecutor { boolean isFusionEnabled() { return FusionHelper.isFusionEnabled(session) } + + @Override + String getArrayIndexName() { + return 'SLURM_ARRAY_TASK_ID' + } + + @Override + int getArrayIndexStart() { + return 0 + } + + @Override + String getArrayTaskId(String jobId, int index) { + assert jobId, "Missing 'jobId' argument" + return "${jobId}_${index}" + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SupportedScriptTypes.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SupportedScriptTypes.groovy index 20b3d5d88e..e755203ceb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SupportedScriptTypes.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SupportedScriptTypes.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/TaskArrayExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/TaskArrayExecutor.groovy new file mode 100644 index 0000000000..b85990d652 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/executor/TaskArrayExecutor.groovy @@ -0,0 +1,87 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.executor + +import java.nio.file.FileSystems +import java.nio.file.Path + +import groovy.transform.CompileStatic +import nextflow.fusion.FusionHelper +import nextflow.processor.TaskHandler +import nextflow.processor.TaskRun +/** + * Interface for executors that support job arrays. + * + * @author Ben Sherman + */ +@CompileStatic +interface TaskArrayExecutor { + + String getName() + + Path getWorkDir() + + void submit( TaskRun task ) + + TaskHandler createTaskHandler(TaskRun task) + + boolean isFusionEnabled() + + /** + * Get the environment variable name that provides the array index of a task. + */ + String getArrayIndexName() + + /** + * Get the start of the job array index range. + */ + int getArrayIndexStart() + + /** + * Get the name of a child job based on the array job name + * and child index. + */ + String getArrayTaskId(String jobId, int index) + + default boolean isWorkDirDefaultFS() { + getWorkDir().fileSystem== FileSystems.default + } + + /** + * Get a {@link TaskHandler} work directory for the task array resolution + * + * @param handler + * @return + */ + default String getArrayWorkDir(TaskHandler handler) { + return isFusionEnabled() + ? FusionHelper.toContainerMount(handler.task.workDir).toString() + : handler.task.workDir.toUriString() + } + + default String getArrayLaunchCommand(String taskDir) { + if( isFusionEnabled() ) { + return "bash ${taskDir}/${TaskRun.CMD_RUN}" + } + else if( isWorkDirDefaultFS() ) { + return "bash ${taskDir}/${TaskRun.CMD_RUN} 2>&1 > ${taskDir}/${TaskRun.CMD_LOG}" + } + else { + throw new IllegalStateException("Executor ${getName()} does not support array jobs") + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalExecutor.groovy index aa0f1f4f5c..1e98eab1a7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalTaskHandler.groovy index 6c02c57e01..052b10b708 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalTaskHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/local/LocalTaskHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/local/NativeTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/local/NativeTaskHandler.groovy index 8b8a24f94b..910a4248e0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/local/NativeTaskHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/local/NativeTaskHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package nextflow.executor.local +import java.lang.reflect.InvocationTargetException import java.util.concurrent.Callable import java.util.concurrent.Future @@ -91,11 +92,15 @@ class NativeTaskHandler extends TaskHandler { boolean checkIfCompleted() { if( isRunning() && result.isDone() ) { status = TaskStatus.COMPLETED - if( result.get() instanceof Throwable ) { - task.error = (Throwable)result.get() + final ret = result.get() + if( ret instanceof InvocationTargetException ) { + task.error = ret.cause + } + else if( ret instanceof Throwable ) { + task.error = (Throwable)ret } else { - task.stdout = result.get() + task.stdout = ret } return true } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/res/AcceleratorResource.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/res/AcceleratorResource.groovy index c87a885b25..24e5233b33 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/res/AcceleratorResource.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/res/AcceleratorResource.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/res/DiskResource.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/res/DiskResource.groovy index 3ba0104cb1..25939c7e90 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/res/DiskResource.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/res/DiskResource.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,6 +45,10 @@ class DiskResource { this.type = opts.type as String } + DiskResource withRequest(MemoryUnit value) { + return new DiskResource(request: value, type: this.type) + } + private static MemoryUnit toMemoryUnit( value ) { if( value instanceof MemoryUnit ) return (MemoryUnit)value diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy index da9c575b0e..7fc5067620 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy index 493aee936b..896381dc25 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy index e68ed185c1..b80bad86bd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy index 2ebc20ae87..ba68ef0390 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy index a5382379d8..6f3dcd2b75 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index 017149e28a..9d7b02558f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index 729e500004..8ef6765faf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index 801ac412b2..f21a34d76b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy index 7f191ae8d1..c1511aeb9b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index dd4cdd5b5a..14c775b04e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy index e904834b89..b01fa480b7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DefaultMergeClosure.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DumpHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DumpHelper.groovy index 8c190cf17d..3b04ab8036 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DumpHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DumpHelper.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy index 6e499c048b..2aae01dbab 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupKey.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupKey.groovy index 0a84febb60..8c005efbe4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupKey.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupKey.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy index e51a3b6bc7..660ae6cf63 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy index 25fa2fbbfd..cfcb0390e1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy index 7544b30996..3d9075ba60 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy index 6100c66d6d..e794760b82 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index 4599674176..78e05a0777 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy index 200d37cdcc..561567575b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy index 62ba37d5f2..37e49d8e45 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy index 9accaaf312..57693b6901 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index ceeb26f858..63970a8c55 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy index 1c99956cc8..965a8247eb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index d0bbcbc0d2..12f2e9d896 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,9 @@ import groovyx.gpars.dataflow.DataflowReadChannel import nextflow.Global import nextflow.Session import nextflow.processor.PublishDir +import nextflow.util.CsvWriter /** + * Publish files from a source channel. * * @author Paolo Di Tommaso */ @@ -34,11 +36,13 @@ class PublishOp { private DataflowReadChannel source - private Map opts - private PublishDir publisher - private Path sourceDir + private Path targetDir + + private IndexOpts indexOpts + + private List indexRecords = [] private volatile boolean complete @@ -46,80 +50,151 @@ class PublishOp { PublishOp(DataflowReadChannel source, Map opts) { this.source = source - this.opts = opts ? new LinkedHashMap(opts) : Collections.emptyMap() - - // adapt `to` option - if( this.opts.containsKey('to') ) { - this.opts.path = this.opts.to - this.opts.remove('to') - } - - this.publisher = PublishDir.create(this.opts) + this.publisher = PublishDir.create(opts) + this.targetDir = opts.path as Path + if( opts.index ) + this.indexOpts = new IndexOpts(targetDir, opts.index as Map) } - protected boolean getComplete() { complete } + boolean getComplete() { complete } PublishOp apply() { final events = new HashMap(2) - events.onNext = this.&publish0 - events.onComplete = this.&done0 + events.onNext = this.&onNext + events.onComplete = this.&onComplete DataflowHelper.subscribeImpl(source, events) return this } - protected void publish0(entry) { - log.debug "Publish operator got: $entry" - sourceDir = null - // use a set to avoid duplicates - final result = new HashSet(10) - collectFiles(entry, result) - publisher.apply(result, sourceDir) + protected void onNext(value) { + log.trace "Publish operator received: $value" + final result = collectFiles([:], value) + for( final entry : result ) { + final sourceDir = entry.key + final files = entry.value + publisher.apply(files, sourceDir) + } + + if( indexOpts ) { + final record = indexOpts.mapper != null ? indexOpts.mapper.call(value) : value + final normalized = normalizePaths(record) + log.trace "Normalized record for index file: ${normalized}" + indexRecords << normalized + } } - protected void done0(nope) { - log.debug "Publish operator complete" + protected void onComplete(nope) { + if( indexOpts && indexRecords.size() > 0 && publisher.enabled ) { + log.trace "Saving records to index file: ${indexRecords}" + new CsvWriter(header: indexOpts.header, sep: indexOpts.sep).apply(indexRecords, indexOpts.path) + session.notifyFilePublish(indexOpts.path) + } + + log.trace "Publish operator complete" this.complete = true } - protected void collectFiles(entry, Collection result) { - if( entry instanceof Path ) { - result.add(entry) - if( sourceDir == null ) - sourceDir = getTaskDir(entry) + /** + * Extract files from a received value for publishing. + * + * @param result + * @param value + */ + protected Map> collectFiles(Map> result, value) { + if( value instanceof Path ) { + final sourceDir = getTaskDir(value) + if( sourceDir !in result ) + result[sourceDir] = new HashSet(10) + result[sourceDir] << value + } + else if( value instanceof Collection ) { + for( final el : value ) + collectFiles(result, el) } - else if( entry instanceof List ) { - for( def x : entry ) { - collectFiles(x, result) + return result + } + + /** + * Normalize the paths in a record by converting + * work directory paths to publish paths. + * + * @param value + */ + protected Object normalizePaths(value) { + if( value instanceof Path ) { + return List.of(value.getBaseName(), normalizePath(value)) + } + + if( value instanceof Collection ) { + return value.collect { el -> + if( el instanceof Path ) + return normalizePath(el) + if( el instanceof Collection ) + return normalizePaths(el) + return el + } + } + + if( value instanceof Map ) { + return value.collectEntries { k, v -> + if( v instanceof Path ) + return List.of(k, normalizePath(v)) + if( v instanceof Collection ) + return List.of(k, normalizePaths(v)) + return List.of(k, v) } } + + throw new IllegalArgumentException("Index file record must be a list, map, or file: ${value} [${value.class.simpleName}]") + } + + private Path normalizePath(Path path) { + final sourceDir = getTaskDir(path) + return targetDir.resolve(sourceDir.relativize(path)) } /** - * Given a path try to infer the task directory to which the path below - * ie. the directory starting with a workflow work dir and having at lest - * two sub-directories eg work-dir/xx/yyyyyy/etc + * Try to infer the parent task directory to which a path belongs. It + * should be a directory starting with a session work dir and having + * at lest two sub-directories, e.g. work/ab/cdef/etc * * @param path - * @return */ protected Path getTaskDir(Path path) { - if( path==null ) + if( path == null ) return null - def result = getTaskDir0(path, session.workDir) - if( result == null ) - result = getTaskDir0(path, session.bucketDir) - return result + return getTaskDir0(path, session.workDir.resolve('tmp')) + ?: getTaskDir0(path, session.workDir) + ?: getTaskDir0(path, session.bucketDir) } private Path getTaskDir0(Path file, Path base) { - if( base==null ) + if( base == null ) return null if( base.fileSystem != file.fileSystem ) return null final len = base.nameCount - if( file.startsWith(base) && file.getNameCount()>len+2 ) + if( file.startsWith(base) && file.getNameCount() > len+2 ) return base.resolve(file.subpath(len,len+2)) return null } + static class IndexOpts { + Path path + Closure mapper + def /* boolean | List */ header = false + String sep = ',' + + IndexOpts(Path targetDir, Map opts) { + this.path = targetDir.resolve(opts.path as String) + + if( opts.mapper ) + this.mapper = opts.mapper as Closure + if( opts.header != null ) + this.header = opts.header + if( opts.sep ) + this.sep = opts.sep as String + } + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy index bb948b4b75..97c58ee2da 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy index d5eeb24b00..db0e58462e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy index 836469004a..7ee4be8f9f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy index bcf8b48bd2..f7bf259d0a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy index 7acb33e9ba..15d8527c51 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index 1957b231d9..163415b3f2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy index 04634674c6..e9c26a61f8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy index 558c187f0d..7ec5e49dd2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy index a5872c166b..cb9ebefc6b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/DirListener.groovy b/modules/nextflow/src/main/groovy/nextflow/file/DirListener.groovy index 0ceb1295a7..2bac280727 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/DirListener.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/DirListener.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/DirWatcher.groovy b/modules/nextflow/src/main/groovy/nextflow/file/DirWatcher.groovy index 8990f3785d..174d83b1ac 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/DirWatcher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/DirWatcher.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/DirWatcherV2.groovy b/modules/nextflow/src/main/groovy/nextflow/file/DirWatcherV2.groovy index 10a79add2d..d34b9ef13d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/DirWatcherV2.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/DirWatcherV2.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy index de20b47cd3..189828ed63 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/FileCollector.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/FilePorter.groovy b/modules/nextflow/src/main/groovy/nextflow/file/FilePorter.groovy index e2a044bc60..881f7ff6a4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/FilePorter.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/FilePorter.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -333,7 +333,7 @@ class FilePorter { } private String fmtError(Path filePath, Exception e) { - def message = "Can't stage file ${filePath.toUri().toString()}" + def message = "Can't stage file ${FilesEx.toUriString(filePath)}" if( e instanceof NoSuchFileException ) message += " -- file does not exist" else if( e.message ) diff --git a/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy b/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy index 6a76ab484c..0a484bff9f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/SequentialFileStore.groovy b/modules/nextflow/src/main/groovy/nextflow/file/SequentialFileStore.groovy index 3d323d40fe..ac095cca32 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/SequentialFileStore.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/SequentialFileStore.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/SimpleFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/file/SimpleFileCollector.groovy index ea9990ce13..296c346775 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/SimpleFileCollector.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/SimpleFileCollector.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,14 +15,15 @@ */ package nextflow.file + import java.nio.file.Files import java.nio.file.Path -import java.nio.file.StandardCopyOption import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentMap import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.extension.FilesEx /** * Helper class used to aggregate values having the same key * to files @@ -94,16 +95,13 @@ class SimpleFileCollector extends FileCollector { @Override void saveFile( Closure closure ) { - def result = [] Iterator itr = cache.values().iterator() while( itr.hasNext() ) { - def item = itr.next() - def target = closure.call(item.getName()) - result << Files.move(item, target, StandardCopyOption.REPLACE_EXISTING) + final item = itr.next() + final target = closure.call(item.getName()) + FilesEx.moveTo(item, target) itr.remove() } - } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/file/SlurperEx.groovy b/modules/nextflow/src/main/groovy/nextflow/file/SlurperEx.groovy index 8152fb16e9..04e9da23f2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/SlurperEx.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/SlurperEx.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/file/SortFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/file/SortFileCollector.groovy index dd6982b296..130c78bbcb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/SortFileCollector.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/SortFileCollector.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionAwareTask.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionAwareTask.groovy index d89c462283..d75c8f44fc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionAwareTask.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionAwareTask.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy index e67006cb14..228f3f23cd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,15 @@ package nextflow.fusion + +import java.util.regex.Pattern + import groovy.transform.CompileStatic import groovy.transform.Memoized import nextflow.Global +import nextflow.Session import nextflow.SysEnv import nextflow.util.MemoryUnit - /** * Model Fusion config options * @@ -31,12 +34,14 @@ import nextflow.util.MemoryUnit @CompileStatic class FusionConfig { - final static public String DEFAULT_FUSION_AMD64_URL = 'https://fusionfs.seqera.io/releases/v2.2-amd64.json' - final static public String DEFAULT_FUSION_ARM64_URL = 'https://fusionfs.seqera.io/releases/v2.2-arm64.json' + final static public String DEFAULT_FUSION_AMD64_URL = 'https://fusionfs.seqera.io/releases/v2.3-amd64.json' + final static public String DEFAULT_FUSION_ARM64_URL = 'https://fusionfs.seqera.io/releases/v2.3-arm64.json' final static public String DEFAULT_TAGS = "[.command.*|.exitcode|.fusion.*](nextflow.io/metadata=true),[*](nextflow.io/temporary=true)" final static public String FUSION_PATH = '/usr/bin/fusion' + final static private Pattern VERSION_JSON = ~/https:\/\/.*\/releases\/v(\d+(?:\.\w+)*)-(\w*)\.json$/ + final private Boolean enabled final private String containerConfigUrl @Deprecated final private Boolean exportAwsAccessKeys @@ -99,8 +104,27 @@ class FusionConfig { return createConfig0(Global.config?.fusion as Map ?: Collections.emptyMap(), SysEnv.get()) } + static FusionConfig getConfig(Session session) { + return createConfig0(session.config?.fusion as Map ?: Collections.emptyMap(), SysEnv.get()) + } + @Memoized static private FusionConfig createConfig0(Map config, Map env) { new FusionConfig(config, env) } + + protected String retrieveFusionVersion(String url) { + if( !url ) + return null + final matcher_json = VERSION_JSON.matcher(url) + if( matcher_json.matches() ) + return matcher_json.group(1) + return null + } + + String version() { + return enabled + ? retrieveFusionVersion(this.containerConfigUrl ?: DEFAULT_FUSION_AMD64_URL) + : null + } } diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy index 20cb399847..28b10aa4c5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnv.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy index b31e11e5d9..65cb1f8ca0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionEnvProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionHelper.groovy index 4d852ee983..33c6f812ed 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionHelper.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -90,4 +90,8 @@ class FusionHelper { return Path.of(result) } + static Path toContainerMount(Path path) { + toContainerMount(path, path.scheme) + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy index 639f976174..9e96262f0b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/fusion/FusionScriptLauncher.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sConfig.groovy index aed5d977e3..0e812d2f4f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -204,6 +204,10 @@ class K8sConfig implements Map { podOptions.volumeClaims.collect { it.mountPath } } + boolean cpuLimitsEnabled() { + target.cpuLimits ?: false + } + /** * Find a volume claim name given the mount path * diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sDriverLauncher.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sDriverLauncher.groovy index 3a1b8823b6..c50cef627d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sDriverLauncher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sDriverLauncher.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy index d447de5031..fca02dc49e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sTaskHandler.groovy index 8beb0b4800..a021a085c4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sTaskHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sTaskHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -159,7 +159,8 @@ class K8sTaskHandler extends TaskHandler implements FusionAwareTask { } protected String getSyntheticPodName(TaskRun task) { - "nf-${task.hash}" + final suffix = System.currentTimeMillis().toString().md5()[-5..-1] + return "nf-${task.hash}-${suffix}" } protected String getOwner() { OWNER } @@ -192,6 +193,10 @@ class K8sTaskHandler extends TaskHandler implements FusionAwareTask { return executor.getK8sConfig().entrypointOverride() } + protected boolean cpuLimitsEnabled() { + return executor.getK8sConfig().cpuLimitsEnabled() + } + protected Map newSubmitRequest0(TaskRun task, String imageName) { final launcher = getSubmitCommand(task) @@ -206,6 +211,7 @@ class K8sTaskHandler extends TaskHandler implements FusionAwareTask { .withLabels(getLabels(task)) .withAnnotations(getAnnotations()) .withPodOptions(getPodOptions()) + .withCpuLimits(cpuLimitsEnabled()) // when `entrypointOverride` is false the launcher is run via `args` instead of `command` // to not override the container entrypoint diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sWrapperBuilder.groovy index 0b5ad75a98..a79ab0f846 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sWrapperBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/ClientConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/client/ClientConfig.groovy index 7b5d0341c8..646c48e29c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/ClientConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/ClientConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/ConfigDiscovery.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/client/ConfigDiscovery.groovy index 15d1380fb9..cda8d7ca9d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/ConfigDiscovery.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/ConfigDiscovery.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sClient.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sClient.groovy index 14e3b75d95..6fe1f75a19 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sClient.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sClient.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseApi.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseApi.groovy index 8f031f5794..89b20f3bff 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseApi.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseApi.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseException.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseException.groovy index 45b5b24cac..35e60b96b8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseJson.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseJson.groovy index bf4b53fdba..b2a11e7c93 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseJson.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sResponseJson.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/PodUnschedulableException.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/client/PodUnschedulableException.groovy index fb62fc920f..df6ef6c84d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/PodUnschedulableException.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/PodUnschedulableException.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/client/SSLUtils.java b/modules/nextflow/src/main/groovy/nextflow/k8s/client/SSLUtils.java index 9450f18f2a..208ae0c089 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/client/SSLUtils.java +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/client/SSLUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodEnv.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodEnv.groovy index 157e111fac..9edb1c8704 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodEnv.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodEnv.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodHostMount.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodHostMount.groovy index 949a10ca06..460f2f9d8d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodHostMount.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodHostMount.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountConfig.groovy index 968348b767..704c2b6a33 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountCsiEphemeral.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountCsiEphemeral.groovy index 0fac436b75..3c4ed9364f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountCsiEphemeral.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountCsiEphemeral.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountEmptyDir.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountEmptyDir.groovy index 1a81f971ce..ee407ad2d9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountEmptyDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountEmptyDir.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountSecret.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountSecret.groovy index 71465ab6ec..7e0124f6ef 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountSecret.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodMountSecret.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodNodeSelector.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodNodeSelector.groovy index d92c61a95e..c21944c80f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodNodeSelector.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodNodeSelector.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodOptions.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodOptions.groovy index 4c1d0000e8..301433ceab 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodOptions.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodOptions.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,6 +69,8 @@ class PodOptions { private Boolean privileged private String schedulerName + + private Integer ttlSecondsAfterFinished PodOptions( List options=null ) { int size = options ? options.size() : 0 @@ -161,6 +163,9 @@ class PodOptions { else if( entry.schedulerName ) { this.schedulerName = entry.schedulerName } + else if( entry.ttlSecondsAfterFinished instanceof Integer ) { + this.ttlSecondsAfterFinished = entry.ttlSecondsAfterFinished as Integer + } else throw new IllegalArgumentException("Unknown pod options: $entry") } @@ -228,7 +233,9 @@ class PodOptions { List getTolerations() { tolerations } Boolean getPrivileged() { privileged } - + + Integer getTtlSecondsAfterFinished() { ttlSecondsAfterFinished } + PodOptions plus( PodOptions other ) { def result = new PodOptions() @@ -261,10 +268,7 @@ class PodOptions { result.volumeClaims.addAll( other.volumeClaims ) // sec context - if( other.securityContext ) - result.securityContext = other.securityContext - else - result.securityContext = securityContext + result.securityContext = other.securityContext ?: this.securityContext // node selector result.nodeSelector = other.nodeSelector ?: this.nodeSelector @@ -273,16 +277,10 @@ class PodOptions { result.affinity = other.affinity ?: this.affinity // pull policy - if (other.imagePullPolicy) - result.imagePullPolicy = other.imagePullPolicy - else - result.imagePullPolicy = imagePullPolicy + result.imagePullPolicy = other.imagePullPolicy ?: this.imagePullPolicy // image secret - if (other.imagePullSecret) - result.imagePullSecret = other.imagePullSecret - else - result.imagePullSecret = imagePullSecret + result.imagePullSecret = other.imagePullSecret ?: this.imagePullSecret // labels result.labels.putAll(labels) @@ -307,6 +305,9 @@ class PodOptions { // scheduler name result.schedulerName = other.schedulerName ?: this.schedulerName + // ttl seconds after finished (job) + result.ttlSecondsAfterFinished = other.ttlSecondsAfterFinished!=null ? other.ttlSecondsAfterFinished : this.ttlSecondsAfterFinished + return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSecurityContext.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSecurityContext.groovy index 7c91f48fd8..2e8b6d4fb2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSecurityContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSecurityContext.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSpecBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSpecBuilder.groovy index 24c56bc308..82d661725c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSpecBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodSpecBuilder.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -76,6 +76,8 @@ class PodSpecBuilder { Integer cpus + boolean cpuLimits + String memory String disk @@ -120,6 +122,8 @@ class PodSpecBuilder { String schedulerName + Integer ttlSecondsAfterFinished + /** * @return A sequential volume unique identifier */ @@ -165,14 +169,14 @@ class PodSpecBuilder { PodSpecBuilder withCommand( cmd ) { if( cmd==null ) return this assert cmd instanceof List || cmd instanceof CharSequence, "Missing or invalid K8s command parameter: $cmd" - this.command = cmd instanceof List ? cmd : ['/bin/bash','-c', cmd.toString()] + this.command = cmd instanceof List ? cmd as List : ['/bin/bash','-c', cmd.toString()] return this } PodSpecBuilder withArgs( args ) { if( args==null ) return this assert args instanceof List || args instanceof CharSequence, "Missing or invalid K8s args parameter: $args" - this.args = args instanceof List ? args : ['/bin/bash','-c', args.toString()] + this.args = args instanceof List ? args as List : ['/bin/bash','-c', args.toString()] return this } @@ -181,6 +185,11 @@ class PodSpecBuilder { return this } + PodSpecBuilder withCpuLimits(boolean cpuLimits) { + this.cpuLimits = cpuLimits + return this + } + PodSpecBuilder withMemory(String mem) { this.memory = mem return this @@ -378,6 +387,9 @@ class PodSpecBuilder { privileged = opts.privileged // -- scheduler name schedulerName = opts.schedulerName + // -- ttl seconds after finished (job) + if( opts.ttlSecondsAfterFinished != null ) + ttlSecondsAfterFinished = opts.ttlSecondsAfterFinished return this } @@ -576,25 +588,29 @@ class PodSpecBuilder { Map buildAsJob() { final pod = build() + final spec = [ + backoffLimit: 0, + template: [ + metadata: pod.metadata, + spec: pod.spec + ] + ] + + if( ttlSecondsAfterFinished != null ) + spec.ttlSecondsAfterFinished = ttlSecondsAfterFinished return [ apiVersion: 'batch/v1', kind: 'Job', metadata: pod.metadata, - spec: [ - backoffLimit: 0, - template: [ - metadata: pod.metadata, - spec: pod.spec - ] - ] + spec: spec ] } @PackageScope Map addResourcesLimits(Map limits, Map result) { if( result == null ) - result = new LinkedHashMap(10) + result = new LinkedHashMap(2) final limits0 = result.limits as Map ?: new LinkedHashMap(10) limits0.putAll( limits ) @@ -606,11 +622,17 @@ class PodSpecBuilder { @PackageScope Map addCpuResources(Integer cpus, Map res) { if( res == null ) - res = [:] + res = new LinkedHashMap(2) - final req = res.requests as Map ?: new LinkedHashMap<>(10) - req.cpu = cpus - res.requests = req + final requests0 = res.requests as Map ?: new LinkedHashMap<>(10) + requests0.cpu = cpus + res.requests = requests0 + + if( cpuLimits ) { + final limits0 = res.limits as Map ?: new LinkedHashMap(10) + limits0.cpu = cpus + res.limits = limits0 + } return res } @@ -618,7 +640,7 @@ class PodSpecBuilder { @PackageScope Map addMemoryResources(String memory, Map res) { if( res == null ) - res = new LinkedHashMap(10) + res = new LinkedHashMap(2) final req = res.requests as Map ?: new LinkedHashMap(10) req.memory = memory @@ -634,7 +656,7 @@ class PodSpecBuilder { @PackageScope Map addDiskResources(String diskRequest, Map res) { if( res == null ) - res = new LinkedHashMap(10) + res = new LinkedHashMap(2) final req = res.requests as Map ?: new LinkedHashMap(10) req.'ephemeral-storage' = diskRequest diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodVolumeClaim.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodVolumeClaim.groovy index 021e412bf6..66afd33890 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodVolumeClaim.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/PodVolumeClaim.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/model/ResourceType.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/model/ResourceType.groovy index 64fe25517d..6315f00361 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/model/ResourceType.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/model/ResourceType.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/Attachment.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/Attachment.groovy index 83959798b2..d6c897101a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/mail/Attachment.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/mail/Attachment.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/Mail.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/Mail.groovy index e2fedd1290..ddc9032166 100644 --- a/modules/nextflow/src/main/groovy/nextflow/mail/Mail.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/mail/Mail.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy index d7c09b13de..9d4264fce1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/ChannelFactoryInstance.groovy b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/ChannelFactoryInstance.groovy index 32fce322e4..1adff3a49c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/ChannelFactoryInstance.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/ChannelFactoryInstance.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Factory.groovy b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Factory.groovy index f700dfa474..32773b6503 100644 --- a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Factory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Factory.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Function.groovy b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Function.groovy index fdf511bbfa..3e8fabdec2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Function.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Function.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Operator.groovy b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Operator.groovy index 3727915199..6a657f564c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Operator.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/Operator.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionMethod.groovy b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionMethod.groovy index 9d8e8026f4..fd8efc7ab3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionMethod.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionMethod.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionPoint.groovy b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionPoint.groovy index b1f9f1fe2f..c6ffd3c8ba 100644 --- a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionPoint.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionPoint.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionProvider.groovy index f24de108a5..0ca983a6c1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/plugin/extension/PluginExtensionProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -65,8 +65,6 @@ class PluginExtensionProvider implements ExtensionProvider { */ final private Map factoryExtensions = new HashMap<>() - private List channelExtensionPoints - private Set OPERATOR_NAMES static PluginExtensionProvider INSTANCE() { diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/BatchContext.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/BatchContext.groovy index 4f2358c24b..cf5b40b394 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/BatchContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/BatchContext.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/BatchHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/BatchHandler.groovy index 0bf815fae0..b84d977e88 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/BatchHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/BatchHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/ErrorStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/ErrorStrategy.groovy index 2533adff02..47ef454682 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/ErrorStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/ErrorStrategy.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy index 8fb48e4515..7b1a8e01d0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/ForwardClosure.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy index 3c26ca54f6..277bfc746d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/InvokeTaskAdapter.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/LocalPollingMonitor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/LocalPollingMonitor.groovy index be9e19aef1..d9dbda638a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/LocalPollingMonitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/LocalPollingMonitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/ParallelPollingMonitor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/ParallelPollingMonitor.groovy index 64b8ee52b6..07af8f22c2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/ParallelPollingMonitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/ParallelPollingMonitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,7 +55,7 @@ class ParallelPollingMonitor extends TaskPollingMonitor { @Override protected boolean canSubmit(TaskHandler handler) { - return super.canSubmit(handler) && semaphore?.tryAcquire() + return super.canSubmit(handler) && (semaphore == null || semaphore.tryAcquire()) } protected RateLimiter createSubmitRateLimit() { @@ -86,7 +86,7 @@ class ParallelPollingMonitor extends TaskPollingMonitor { if( !session.success ) return // ignore error when the session has been interrupted handleException(handler, e) - session.notifyTaskComplete(handler) + notifyTaskComplete(handler) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index 039230ca60..67e1fb813d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,13 +27,16 @@ import java.nio.file.Path import java.nio.file.PathMatcher import java.nio.file.SimpleFileVisitor import java.nio.file.attribute.BasicFileAttributes +import java.time.temporal.ChronoUnit import java.util.concurrent.ExecutorService -import java.util.regex.Pattern +import dev.failsafe.Failsafe +import dev.failsafe.RetryPolicy +import dev.failsafe.event.EventListener +import dev.failsafe.event.ExecutionAttemptedEvent import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode -import groovy.transform.Memoized import groovy.transform.PackageScope import groovy.transform.ToString import groovy.util.logging.Slf4j @@ -44,7 +47,11 @@ import nextflow.extension.FilesEx import nextflow.file.FileHelper import nextflow.file.TagAwareFile import nextflow.fusion.FusionHelper +import nextflow.util.HashBuilder import nextflow.util.PathTrie + +import static nextflow.util.CacheHelper.HashMode + /** * Implements the {@code publishDir} directory. It create links or copies the output * files of a given task to a user specified directory. @@ -57,8 +64,6 @@ import nextflow.util.PathTrie @CompileStatic class PublishDir { - final static private Pattern FUSION_PATH_REGEX = ~/^\/fusion\/([^\/]+)\/(.*)/ - enum Mode { SYMLINK, LINK, COPY, MOVE, COPY_NO_FOLLOW, RELLINK } private Map makeCache = new HashMap<>() @@ -73,7 +78,7 @@ class PublishDir { /** * Whether to overwrite existing files */ - Boolean overwrite + def /* Boolean | String */ overwrite /** * The publish {@link Mode} @@ -98,7 +103,7 @@ class PublishDir { /** * Trow an exception in case publish fails */ - boolean failOnError = false + boolean failOnError = true /** * Tags to be associated to the target file @@ -117,6 +122,8 @@ class PublishDir { */ private String storageClass + private PublishRetryConfig retryConfig + private PathMatcher matcher private FileSystem sourceFileSystem @@ -136,10 +143,6 @@ class PublishDir { return task?.getName() } - protected Map getTaskInputs() { - return task ? task.getInputFilesMap() : Map.of() - } - void setPath( def value ) { final resolved = value instanceof Closure ? value.call() : value if( resolved instanceof String || resolved instanceof GString ) @@ -195,7 +198,7 @@ class PublishDir { result.pattern = params.pattern if( params.overwrite != null ) - result.overwrite = Boolean.parseBoolean(params.overwrite.toString()) + result.overwrite = params.overwrite if( params.saveAs ) result.saveAs = (Closure) params.saveAs @@ -223,6 +226,9 @@ class PublishDir { protected void apply0(Set files) { assert path + final retryOpts = session.config.navigate('nextflow.publish.retryPolicy') as Map ?: Collections.emptyMap() + this.retryConfig = new PublishRetryConfig(retryOpts) + createPublishDir() validatePublishMode() @@ -362,42 +368,55 @@ class PublishDir { protected void safeProcessPath(Path source, Path target) { try { - processPath(source, target) + // publish each file in the directory tree + if( Files.isDirectory(source) ) { + Files.walkFileTree(source, new SimpleFileVisitor() { + FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) { + final targetFile = target.resolve(source.relativize(sourceFile).toString()) + retryableProcessFile(sourceFile, targetFile) + FileVisitResult.CONTINUE + } + }) + } + + // otherwise publish file directly + else { + retryableProcessFile(source, target) + } } catch( Throwable e ) { - log.warn "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- See log file for details", e - if( NF.strictMode || failOnError ) { + final msg = "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- See log file for details" + final shouldFail = NF.strictMode || failOnError + if( shouldFail ) { + log.error(msg,e) session?.abort(e) } + else + log.warn(msg,e) } } - protected void processPath(Path source, Path target) { - - // publish each file in the directory tree - if( Files.isDirectory(source) ) - Files.walkFileTree(source, new SimpleFileVisitor() { - FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) { - final targetFile = target.resolve(source.relativize(sourceFile).toString()) - processFile(sourceFile, targetFile) - FileVisitResult.CONTINUE - } - }) - - // otherwise publish file directly - else - processFile(source, target) + protected void retryableProcessFile(Path source, Path target) { + final listener = new EventListener() { + @Override + void accept(ExecutionAttemptedEvent event) throws Throwable { + log.debug "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- attempt: ${event.attemptCount}; reason: ${event.lastFailure.message}" + } + } + final retryPolicy = RetryPolicy.builder() + .handle(Exception) + .withBackoff(retryConfig.delay.toMillis(), retryConfig.maxDelay.toMillis(), ChronoUnit.MILLIS) + .withMaxAttempts(retryConfig.maxAttempts) + .withJitter(retryConfig.jitter) + .onRetry(listener) + .build() + Failsafe + .with( retryPolicy ) + .get { it -> processFile(source, target) } } protected void processFile( Path source, Path destination ) { - // resolve Fusion symlink if applicable - if( FusionHelper.isFusionEnabled(session) ) { - final inputs = getTaskInputs() - if( source.name in inputs ) - source = resolveFusionLink(inputs[source.name]) - } - // create target dirs if required makeDirs(destination.parent) @@ -405,15 +424,17 @@ class PublishDir { processFileImpl(source, destination) } catch( FileAlreadyExistsException e ) { - if( checkIsSameRealPath(source, destination) ) - return + // don't copy source path if target is identical, but still emit the publish event + // see also https://github.com/nextflow-io/nf-prov/issues/22 + final sameRealPath = checkIsSameRealPath(source, destination) + // make sure destination and source does not overlap // see https://github.com/nextflow-io/nextflow/issues/2177 - if( checkSourcePathConflicts(destination)) + if( !sameRealPath && checkSourcePathConflicts(destination) ) return - + // overwrite only if explicitly enabled or destination is stale - if( overwrite || (overwrite == null && source.getChecksum() != destination.getChecksum()) ) { + if( !sameRealPath && shouldOverwrite(source, destination) ) { FileHelper.deletePath(destination) processFileImpl(source, destination) } @@ -422,29 +443,6 @@ class PublishDir { notifyFilePublish(destination, source) } - /** - * Resolve a Fusion symlink by following the .fusion.symlinks - * file in the task directory until the original file is reached. - * - * @param file - */ - protected Path resolveFusionLink(Path file) { - while( file.name in getFusionLinks(file.parent) ) - file = file.text.replaceFirst(FUSION_PATH_REGEX) { _, scheme, path -> "${scheme}://${path}" } as Path - return file - } - - @Memoized - protected List getFusionLinks(Path workDir) { - try { - final file = workDir.resolve('.fusion.symlinks') - return file.text.tokenize('\n') - } - catch( NoSuchFileException e ) { - return List.of() - } - } - private String real0(Path p) { try { // resolve symlink if it's file in the default (posix) file system @@ -497,6 +495,17 @@ class PublishDir { return !mode || mode == Mode.SYMLINK || mode == Mode.RELLINK } + protected boolean shouldOverwrite(Path source, Path target) { + if( overwrite instanceof Boolean ) + return overwrite + + final hashMode = HashMode.of(overwrite) ?: HashMode.DEFAULT() + final sourceHash = HashBuilder.hashPath(source, source.parent, hashMode) + final targetHash = HashBuilder.hashPath(target, target.parent, hashMode) + log.trace "comparing source and target with mode=${overwrite}, source=${sourceHash}, target=${targetHash}, should overwrite=${sourceHash != targetHash}" + return sourceHash != targetHash + } + protected void processFileImpl( Path source, Path destination ) { log.trace "publishing file: $source -[$mode]-> $destination" diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishRetryConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishRetryConfig.groovy new file mode 100644 index 0000000000..860a849678 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishRetryConfig.groovy @@ -0,0 +1,53 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.processor + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import nextflow.util.Duration + +/** + * Models retry policy configuration for publishing outputs + * + * @author Ben Sherman + */ +@ToString(includePackage = false, includeNames = true) +@EqualsAndHashCode +@CompileStatic +class PublishRetryConfig { + Duration delay = Duration.of('350ms') + Duration maxDelay = Duration.of('90s') + int maxAttempts = 5 + double jitter = 0.25 + + PublishRetryConfig() { + this(Collections.emptyMap()) + } + + PublishRetryConfig(Map config) { + if( config.delay ) + delay = config.delay as Duration + if( config.maxDelay ) + maxDelay = config.maxDelay as Duration + if( config.maxAttempts ) + maxAttempts = config.maxAttempts as int + if( config.jitter ) + jitter = config.jitter as double + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/StateObj.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/StateObj.groovy index 069055065b..c6bc27b3b8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/StateObj.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/StateObj.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy new file mode 100644 index 0000000000..b5a40db98d --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayCollector.groovy @@ -0,0 +1,204 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.processor + +import java.nio.file.Files +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.executor.Executor +import nextflow.executor.TaskArrayExecutor +import nextflow.file.FileHelper +import nextflow.util.CacheHelper +import nextflow.util.Escape +/** + * Task monitor that batches tasks and submits them as job arrays + * to an underlying task monitor. + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class TaskArrayCollector { + + /** + * The set of directives which are used by the job array. + */ + private static final List ARRAY_DIRECTIVES = [ + 'accelerator', + 'arch', + 'clusterOptions', + 'cpus', + 'disk', + 'machineType', + 'memory', + 'queue', + 'resourceLabels', + 'resourceLimits', + 'time', + // only needed for container-native executors and/or Fusion + 'container', + 'containerOptions', + ] + + private TaskProcessor processor + + private TaskArrayExecutor executor + + private int arraySize + + private Lock sync = new ReentrantLock() + + private List array + + private boolean closed = false + + TaskArrayCollector(TaskProcessor processor, Executor executor, int arraySize) { + if( executor !instanceof TaskArrayExecutor ) + throw new IllegalArgumentException("Executor '${executor.name}' does not support job arrays") + + this.processor = processor + this.executor = (TaskArrayExecutor)executor + this.arraySize = arraySize + this.array = new ArrayList<>(arraySize) + } + + /** + * Add a task to the current array, and submit the array when it + * reaches the desired size. + * + * @param task + */ + void collect(TaskRun task) { + sync.lock() + try { + // submit task directly if the collector is closed + // or if the task is retried (since it might have dynamic resources) + if( closed || task.config.getAttempt() > 1 ) { + executor.submit(task) + return + } + + // add task to the array + array.add(task) + + // submit job array when it is ready + if( array.size() == arraySize ) { + executor.submit(createTaskArray(array)) + array = new ArrayList<>(arraySize) + } + } + finally { + sync.unlock() + } + } + + /** + * Close the collector, submitting any remaining tasks as a partial job array. + */ + void close() { + sync.lock() + try { + if( array.size() == 1 ) { + executor.submit(array.first()) + } + else if( array.size() > 0 ) { + executor.submit(createTaskArray(array)) + array = null + } + closed = true + } + finally { + sync.unlock() + } + } + + /** + * Create the task run for a job array. + * + * @param tasks + */ + protected TaskArrayRun createTaskArray(List tasks) { + // prepare child job launcher scripts + final handlers = tasks.collect( t -> executor.createTaskHandler(t) ) + for( TaskHandler handler : handlers ) { + handler.prepareLauncher() + } + + // create work directory + final hash = CacheHelper.hasher( tasks.collect( t -> t.getHash().asLong() ) ).hash() + final workDir = FileHelper.getWorkFolder(executor.getWorkDir(), hash) + Files.createDirectories(workDir) + + // create wrapper script + final script = createArrayTaskScript(handlers) + log.debug "Creating task array run >> $workDir\n$script" + + // create config for job array + final rawConfig = new HashMap(ARRAY_DIRECTIVES.size()) + for( final key : ARRAY_DIRECTIVES ) { + final value = processor.config.get(key) + if( value != null ) + rawConfig[key] = value + } + + // create job array + final first = tasks.min( t -> t.index ) + final taskArray = new TaskArrayRun( + id: first.id, + index: first.index, + processor: processor, + type: processor.taskBody.type, + config: new TaskConfig(rawConfig), + context: new TaskContext(processor), + hash: hash, + workDir: workDir, + script: script, + children: handlers + ) + taskArray.config.context = taskArray.context + taskArray.config.process = taskArray.processor.name + taskArray.config.executor = taskArray.processor.executor.name + + return taskArray + } + + /** + * Create the wrapper script for a job array. + * + * @param array + */ + protected String createArrayTaskScript(List array) { + // get work directory and launch command for each task + final workDirs = array.collect( h -> executor.getArrayWorkDir(h) ) + """ + array=( ${workDirs.collect( p -> Escape.path(p) ).join(' ')} ) + export nxf_array_task_dir=${getArrayIndexRef()} + ${executor.getArrayLaunchCommand('$nxf_array_task_dir')} + """.stripIndent().leftTrim() + } + + protected String getArrayIndexRef() { + final name = executor.getArrayIndexName() + final start = executor.getArrayIndexStart() + final index = start > 0 ? "${name} - ${start}" : name + return '${array[' + index + ']}' + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy new file mode 100644 index 0000000000..6d03af83f0 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskArrayRun.groovy @@ -0,0 +1,55 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.processor + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.container.ContainerConfig +import nextflow.executor.TaskArrayExecutor + +/** + * Models a task array, which submits a collection of independent + * tasks with a single submit script. + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class TaskArrayRun extends TaskRun { + + List children + + int getArraySize() { + children.size() + } + + @Override + ContainerConfig getContainerConfig() { + final config = super.getContainerConfig() + final envWhitelist = config.getEnvWhitelist() ?: [] + final executor = (TaskArrayExecutor)processor.getExecutor() + envWhitelist << executor.getArrayIndexName() + config.put('envWhitelist', envWhitelist) + return config + } + + @Override + boolean isContainerEnabled() { + return false + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy index 441c19795e..d1be0cf236 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import groovy.transform.CompileStatic import groovy.transform.PackageScope import nextflow.container.ContainerConfig import nextflow.executor.BashWrapperBuilder +import nextflow.executor.TaskArrayExecutor import nextflow.util.MemoryUnit /** @@ -73,6 +74,8 @@ class TaskBean implements Serializable, Cloneable { List outputEnvNames + Map outputEvals + String beforeScript String afterScript @@ -101,6 +104,12 @@ class TaskBean implements Serializable, Cloneable { Map resourceLabels + String arrayIndexName + + Integer arrayIndexStart + + List arrayWorkDirs + @PackageScope TaskBean() { shell = BashWrapperBuilder.BASH @@ -144,6 +153,7 @@ class TaskBean implements Serializable, Cloneable { // stats this.outputEnvNames = task.getOutputEnvNames() + this.outputEvals = task.getOutputEvals() this.statsEnabled = task.getProcessor().getSession().statsEnabled this.inputFiles = task.getInputFilesMap() @@ -153,6 +163,14 @@ class TaskBean implements Serializable, Cloneable { this.stageOutMode = task.config.getStageOutMode() this.resourceLabels = task.config.getResourceLabels() + + // job array + if( task instanceof TaskArrayRun ) { + final executor = (TaskArrayExecutor)task.getProcessor().getExecutor() + this.arrayIndexName = executor.getArrayIndexName() + this.arrayIndexStart = executor.getArrayIndexStart() + this.arrayWorkDirs = task.children.collect( h -> h.task.workDir ) + } } @Override diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy index c0ed5115c8..4a7b7d09e7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,6 @@ package nextflow.processor -import nextflow.util.CmdLineOptionMap - import static nextflow.processor.TaskProcessor.* import java.nio.file.Path @@ -33,6 +31,7 @@ import nextflow.executor.res.DiskResource import nextflow.k8s.model.PodOptions import nextflow.script.TaskClosure import nextflow.util.CmdLineHelper +import nextflow.util.CmdLineOptionMap import nextflow.util.Duration import nextflow.util.MemoryUnit /** @@ -186,6 +185,10 @@ class TaskConfig extends LazyMap implements Cloneable { return false } + int getArray() { + get('array') as Integer ?: 0 + } + String getBeforeScript() { return get('beforeScript') } @@ -235,8 +238,12 @@ class TaskConfig extends LazyMap implements Cloneable { throw new IllegalArgumentException("Not a valid `ErrorStrategy` value: ${strategy}") } + def getResourceLimit(String directive) { + final limits = get('resourceLimits') as Map + return limits?.get(directive) + } - MemoryUnit getMemory() { + private MemoryUnit getMemory0() { def value = get('memory') if( !value ) @@ -253,7 +260,13 @@ class TaskConfig extends LazyMap implements Cloneable { } } - DiskResource getDiskResource() { + MemoryUnit getMemory() { + final val = getMemory0() + final lim = getResourceLimit('memory') as MemoryUnit + return val && lim && val > lim ? lim : val + } + + private DiskResource getDiskResource0() { def value = get('disk') if( value instanceof Map ) @@ -265,6 +278,12 @@ class TaskConfig extends LazyMap implements Cloneable { return null } + DiskResource getDiskResource() { + final val = getDiskResource0() + final lim = getResourceLimit('disk') as MemoryUnit + return val && lim && val.request > lim ? val.withRequest(lim) : val + } + MemoryUnit getDisk() { getDiskResource()?.getRequest() } @@ -290,10 +309,16 @@ class TaskConfig extends LazyMap implements Cloneable { } - Duration getTime() { + private Duration getTime0() { return getDuration0('time') } + Duration getTime() { + final val = getTime0() + final lim = getResourceLimit('time') as Duration + return val && lim && val > lim ? lim : val + } + Duration getMaxSubmitAwait() { return getDuration0('maxSubmitAwait') } @@ -302,11 +327,17 @@ class TaskConfig extends LazyMap implements Cloneable { get('cpus') != null } - int getCpus() { + private int getCpus0() { final value = get('cpus') value ? value as int : 1 // note: always return at least 1 cpus } + int getCpus() { + final val = getCpus0() + final lim = getResourceLimit('cpus') as Integer + return val && lim && val > lim ? lim : val + } + int getMaxRetries() { def result = get('maxRetries') def defResult = getErrorStrategy() == ErrorStrategy.RETRY ? 1 : 0 @@ -382,10 +413,6 @@ class TaskConfig extends LazyMap implements Cloneable { throw new IllegalArgumentException("Not a valid PublishDir collection [${dirs.getClass().getName()}] $dirs") } - String getClusterOptions() { - return get('clusterOptions') - } - def getContainer() { return get('container') } @@ -401,6 +428,21 @@ class TaskConfig extends LazyMap implements Cloneable { return null } + def getClusterOptions() { + return get('clusterOptions') + } + + String getClusterOptionsAsString() { + final opts = getClusterOptions() + if( opts instanceof CharSequence ) + return opts.toString() + if( opts instanceof Collection ) + return CmdLineHelper.toLine(opts as List) + if( opts != null ) + throw new IllegalArgumentException("Unexpected value for clusterOptions process directive - offending value: $opts") + return null + } + /** * @return Parse the {@code clusterOptions} configuration option and return the entries as a list of values */ diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy index 1ce7ca7846..3217162a5e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskEntry.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskEntry.groovy index ed4ac32362..72771f155e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskEntry.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskEntry.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFault.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFault.groovy index a3f7ae0b42..80b231874f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskFault.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskFault.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskHandler.groovy index 9efb29925a..f7ae9c652d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskHandler.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -88,6 +88,14 @@ abstract class TaskHandler { */ abstract void submit() + /** + * Prepare the launcher script. + * + * This method is optional. If it is not implemented, the launcher script should + * be prepared in the submit() method. + */ + void prepareLauncher() {} + /** * Task status attribute setter. * diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy index 0e88bcc74f..1207a42ec0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskMonitor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskMonitor.groovy index f5b28f71ad..154b40d5c4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskMonitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskMonitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskPath.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskPath.groovy index 1a0cdc4452..131381349c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskPath.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskPath.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskPollingMonitor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskPollingMonitor.groovy index 86e6792627..007d38f255 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskPollingMonitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskPollingMonitor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package nextflow.processor +import java.util.concurrent.ExecutorService import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit import java.util.concurrent.locks.Condition @@ -26,12 +27,15 @@ import com.google.common.util.concurrent.RateLimiter import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Session +import nextflow.SysEnv import nextflow.exception.ProcessSubmitTimeoutException import nextflow.executor.BatchCleanup import nextflow.executor.GridTaskHandler import nextflow.util.Duration import nextflow.util.Threads import nextflow.util.Throttle +import static nextflow.util.SysHelper.dumpThreads + /** * Monitors the queued tasks waiting for their termination * @@ -111,6 +115,11 @@ class TaskPollingMonitor implements TaskMonitor { */ private RateLimiter submitRateLimit + @Lazy + private ExecutorService finalizerPool = { session.taskFinalizerExecutorService() }() + + private boolean enableAsyncFinalizer = SysEnv.get('NXF_ENABLE_ASYNC_FINALIZER','false') as boolean + /** * Create the task polling monitor with the provided named parameters object. *

@@ -192,13 +201,27 @@ class TaskPollingMonitor implements TaskMonitor { * A {@link TaskHandler} instance representing the task to be submitted for execution */ protected void submit(TaskHandler handler) { - // submit the job execution -- throws a ProcessException when submit operation fail - handler.submit() - // note: add the 'handler' into the polling queue *after* the submit operation, - // this guarantees that in the queue are only jobs successfully submitted - runningQueue.add(handler) - // notify task submission - session.notifyTaskSubmit(handler) + if( handler.task instanceof TaskArrayRun ) { + // submit task array + handler.prepareLauncher() + handler.submit() + // add each child task to the running queue + final task = handler.task as TaskArrayRun + for( TaskHandler it : task.children ) { + runningQueue.add(it) + session.notifyTaskSubmit(it) + } + } + else { + // submit the job execution -- throws a ProcessException when submit operation fail + handler.prepareLauncher() + handler.submit() + // note: add the 'handler' into the polling queue *after* the submit operation, + // this guarantees that in the queue are only jobs successfully submitted + runningQueue.add(handler) + // notify task submission + session.notifyTaskSubmit(handler) + } } /** @@ -226,7 +249,7 @@ class TaskPollingMonitor implements TaskMonitor { try{ pendingQueue << handler taskAvail.signal() // signal that a new task is available for execution - session.notifyTaskPending(handler) + notifyTaskPending(handler) log.trace "Scheduled task > $handler" } finally { @@ -403,10 +426,16 @@ class TaskPollingMonitor implements TaskMonitor { protected void pollLoop() { int iteration=0 + int previous=-1 while( true ) { final long time = System.currentTimeMillis() final tasks = new ArrayList(runningQueue) - log.trace "Scheduler queue size: ${tasks.size()} (iteration: ${++iteration})" + final sz = tasks.size() + ++iteration + if( log.isTraceEnabled() && sz!=previous ) { + log.trace "Scheduler queue size: ${sz} (iteration: ${iteration})" + previous = sz + } // check all running tasks for termination checkAllTasks(tasks) @@ -428,10 +457,15 @@ class TaskPollingMonitor implements TaskMonitor { // dump this line every two minutes Throttle.after(dumpInterval) { dumpRunningQueue() + dumpCurrentThreads() } } } + protected dumpCurrentThreads() { + log.trace "Current running threads:\n${dumpThreads()}" + } + protected void dumpRunningQueue() { try { @@ -566,7 +600,7 @@ class TaskPollingMonitor implements TaskMonitor { } catch ( Throwable e ) { handleException(handler, e) - session.notifyTaskComplete(handler) + notifyTaskComplete(handler) } // remove processed handler either on successful submit or failed one (managed by catch section) // when `canSubmit` return false the handler should be retained to be tried in a following iteration @@ -627,18 +661,27 @@ class TaskPollingMonitor implements TaskMonitor { handler.task.error = new ProcessSubmitTimeoutException("Task '${handler.task.lazyName()}' could not be submitted within specified 'maxAwait' time: ${handler.task.config.getMaxSubmitAwait()}") } - // finalize the tasks execution - final fault = handler.task.processor.finalizeTask(handler.task) - - // notify task completion - session.notifyTaskComplete(handler) - - // abort the execution in case of task failure - if (fault instanceof TaskFault) { - session.fault(fault, handler) + // finalize the task asynchronously + if( enableAsyncFinalizer ) { + finalizerPool.submit( ()-> finalizeTask(handler) ) + } + else { + finalizeTask(handler) } } + } + + protected void finalizeTask( TaskHandler handler ) { + // finalize the task execution + final fault = handler.task.processor.finalizeTask(handler.task) + + // notify task completion + session.notifyTaskComplete(handler) + // abort the execution in case of task failure + if( fault instanceof TaskFault ) { + session.fault(fault, handler) + } } /** @@ -688,5 +731,27 @@ class TaskPollingMonitor implements TaskMonitor { return pendingQueue } + protected void notifyTaskPending(TaskHandler handler) { + if( handler.task instanceof TaskArrayRun ) { + final task = handler.task as TaskArrayRun + for( TaskHandler it : task.children ) + session.notifyTaskPending(it) + } + else { + session.notifyTaskPending(handler) + } + } + + protected void notifyTaskComplete(TaskHandler handler) { + if( handler.task instanceof TaskArrayRun ) { + final task = handler.task as TaskArrayRun + for( TaskHandler it : task.children ) + session.notifyTaskComplete(it) + } + else { + session.notifyTaskComplete(handler) + } + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 7cf1d88a09..4e685eaf66 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,6 +61,7 @@ import nextflow.exception.FailedGuardException import nextflow.exception.IllegalArityException import nextflow.exception.MissingFileException import nextflow.exception.MissingValueException +import nextflow.exception.ProcessEvalException import nextflow.exception.ProcessException import nextflow.exception.ProcessFailedException import nextflow.exception.ProcessRetryableException @@ -77,6 +78,8 @@ import nextflow.file.FileHelper import nextflow.file.FileHolder import nextflow.file.FilePatternSplitter import nextflow.file.FilePorter +import nextflow.plugin.Plugins +import nextflow.processor.tip.TaskTipProvider import nextflow.script.BaseScript import nextflow.script.BodyDef import nextflow.script.ProcessConfig @@ -84,6 +87,8 @@ import nextflow.script.ScriptMeta import nextflow.script.ScriptType import nextflow.script.TaskClosure import nextflow.script.bundle.ResourcesBundle +import nextflow.script.params.BaseOutParam +import nextflow.script.params.CmdEvalParam import nextflow.script.params.DefaultOutParam import nextflow.script.params.EachInParam import nextflow.script.params.EnvInParam @@ -104,6 +109,7 @@ import nextflow.util.ArrayBag import nextflow.util.BlankSeparatedList import nextflow.util.CacheHelper import nextflow.util.Escape +import nextflow.util.HashBuilder import nextflow.util.LockManager import nextflow.util.LoggerHelper import nextflow.util.TestOnly @@ -250,6 +256,8 @@ class TaskProcessor { private Boolean isFair0 + private TaskArrayCollector arrayCollector + private CompilerConfiguration compilerConfig() { final config = new CompilerConfiguration() config.addCompilationCustomizers( new ASTTransformationCustomizer(TaskTemplateVarsXform) ) @@ -301,9 +309,12 @@ class TaskProcessor { this.config = config this.taskBody = taskBody this.name = name - this.maxForks = config.maxForks ? config.maxForks as int : 0 + this.maxForks = config.maxForks && config.maxForks>0 ? config.maxForks as int : 0 this.forksCount = maxForks ? new LongAdder() : null this.isFair0 = config.getFair() + + final arraySize = config.getArray() + this.arrayCollector = arraySize > 0 ? new TaskArrayCollector(this, executor, arraySize) : null } /** @@ -364,6 +375,16 @@ class TaskProcessor { boolean hasErrors() { errorCount>0 } + @Memoized + protected TaskTipProvider getTipProvider() { + final provider = Plugins.getPriorityExtensions(TaskTipProvider).find(it-> it.enabled()) + if( !provider ) + throw new IllegalStateException("Unable to find any tip provider") + return provider + } + + boolean isSingleton() { singleton } + /** * Create a "preview" for a task run. This method is only meant for the creation of "mock" task run * to allow the access for the associated {@link TaskConfig} during a pipeline "preview" execution. @@ -783,7 +804,7 @@ class TaskProcessor { int tries = task.failCount +1 while( true ) { - hash = CacheHelper.defaultHasher().newHasher().putBytes(hash.asBytes()).putInt(tries).hash() + hash = HashBuilder.defaultHasher().putBytes(hash.asBytes()).putInt(tries).hash() Path resumeDir = null boolean exists = false @@ -1079,6 +1100,10 @@ class TaskProcessor { formatTaskError( message, error, task ) break + case ProcessEvalException: + formatCommandError( message, error, task ) + break + case FailedGuardException: formatGuardError( message, error as FailedGuardException, task ) break; @@ -1150,6 +1175,35 @@ class TaskProcessor { return action } + final protected List formatCommandError(List message, ProcessEvalException error, TaskRun task) { + // compose a readable error message + message << formatErrorCause(error) + + // - print the executed command + message << "Command executed:\n" + error.command.stripIndent(true)?.trim()?.eachLine { + message << " ${it}" + } + + // - the exit status + message << "\nCommand exit status:\n ${error.status}" + + // - the tail of the process stdout + message << "\nCommand output:" + def lines = error.output.readLines() + if( lines.size() == 0 ) { + message << " (empty)" + } + for( String it : lines ) { + message << " ${stripWorkDir(it, task.workDir)}" + } + + if( task?.workDir ) + message << "\nWork dir:\n ${task.workDirStr}" + + return message + } + final protected List formatGuardError( List message, FailedGuardException error, TaskRun task ) { // compose a readable error message message << formatErrorCause(error) @@ -1229,35 +1283,27 @@ class TaskProcessor { if( task?.workDir ) message << "\nWork dir:\n ${task.workDirStr}" - message << "\nTip: ${getRndTip()}" + message << suggestTip(message) return message } + private String suggestTip(List message) { + try { + return "\nTip: ${getTipProvider().suggestTip(message)}" + } + catch (Exception e) { + log.debug "Unable to get tip for task message: $message", e + return '' + } + } + private static String stripWorkDir(String line, Path workDir) { if( workDir==null ) return line if( workDir.fileSystem != FileSystems.default ) return line return workDir ? line.replace(workDir.toString()+'/','') : line } - static List tips = [ - 'when you have fixed the problem you can continue the execution adding the option `-resume` to the run command line', - "you can try to figure out what's wrong by changing to the process work dir and showing the script file named `${TaskRun.CMD_SCRIPT}`", - "view the complete command output by changing to the process work dir and entering the command `cat ${TaskRun.CMD_OUTFILE}`", - "you can replicate the issue by changing to the process work dir and entering the command `bash ${TaskRun.CMD_RUN}`" - ] - - static Random RND = Random.newInstance() - - /** - * Display a random tip at the bottom of the error report - * - * @return The tip string to display - */ - protected String getRndTip() { - tips[ RND.nextInt( tips.size() ) ] - } - /** * Send a poison pill over all the outputs channel @@ -1290,9 +1336,11 @@ class TaskProcessor { else message = err0(error.cause) + for( String line : message.readLines() ) { + result << ' ' << line << '\n' + } + result - .append(' ') - .append(message) .append('\n') .toString() } @@ -1495,6 +1543,10 @@ class TaskProcessor { collectOutEnvParam(task, (EnvOutParam)param, workDir) break + case CmdEvalParam: + collectOutEnvParam(task, (CmdEvalParam)param, workDir) + break + case DefaultOutParam: task.setOutput(param, DefaultOutParam.Completion.DONE) break @@ -1509,10 +1561,11 @@ class TaskProcessor { task.canBind = true } - protected void collectOutEnvParam(TaskRun task, EnvOutParam param, Path workDir) { + protected void collectOutEnvParam(TaskRun task, BaseOutParam param, Path workDir) { // fetch the output value - final val = collectOutEnvMap(workDir).get(param.name) + final outCmds = param instanceof CmdEvalParam ? task.getOutputEvals() : null + final val = collectOutEnvMap(workDir,outCmds).get(param.name) if( val == null && !param.optional ) throw new MissingValueException("Missing environment variable: $param.name") // set into the output set @@ -1522,25 +1575,56 @@ class TaskProcessor { } + /** + * Parse the `.command.env` file which holds the value for `env` and `cmd` + * output types + * + * @param workDir + * The task work directory that contains the `.command.env` file + * @param outEvals + * A {@link Map} instance containing key-value pairs + * @return + */ + @CompileStatic @Memoized(maxCacheSize = 10_000) - protected Map collectOutEnvMap(Path workDir) { + protected Map collectOutEnvMap(Path workDir, Map outEvals) { final env = workDir.resolve(TaskRun.CMD_ENV).text - final result = new HashMap(50) + final result = new HashMap(50) + Matcher matcher + // `current` represent the current capturing env variable name + String current=null for(String line : env.readLines() ) { - def (k,v) = tokenize0(line) - if (!k) continue - result.put(k,v) + // Opening condition: + // line should match a KEY=VALUE syntax + if( !current && (matcher = (line=~/([a-zA-Z_][a-zA-Z0-9_]*)=(.*)/)) ) { + final k = matcher.group(1) + final v = matcher.group(2) + if (!k) continue + result.put(k,v) + current = k + } + // Closing condition: + // line should match /KEY/ or /KEY/=exit_status + else if( current && (matcher = (line=~/\/${current}\/(?:=exit:(\d+))?/)) ) { + final status = matcher.group(1) as Integer ?: 0 + // when exit status is defined and it is a non-zero, it should be interpreted + // as a failure of the execution of the output command; in this case the variable + // holds the std error message + if( outEvals!=null && status ) { + final cmd = outEvals.get(current) + final out = result[current] + throw new ProcessEvalException("Unable to evaluate output", cmd, out, status) + } + // reset current key + current = null + } + else if( current && line!=null) { + result[current] += '\n' + line + } } return result } - private List tokenize0(String line) { - int p=line.indexOf('=') - return p==-1 - ? List.of(line,'') - : List.of(line.substring(0,p), line.substring(p+1)) - } - /** * Collects the process 'std output' * @@ -2242,7 +2326,10 @@ class TaskProcessor { makeTaskContextStage3(task, hash, folder) // add the task to the collection of running tasks - executor.submit(task) + if( arrayCollector ) + arrayCollector.collect(task) + else + executor.submit(task) } @@ -2336,6 +2423,10 @@ class TaskProcessor { state.update { StateObj it -> it.incCompleted() } } + protected void closeProcess() { + arrayCollector?.close() + } + protected void terminateProcess() { log.trace "<${name}> Sending poison pills and terminating process" sendPoisonPill() @@ -2488,6 +2579,7 @@ class TaskProcessor { // apparently auto if-guard instrumented by @Slf4j is not honoured in inner classes - add it explicitly if( log.isTraceEnabled() ) log.trace "<${name}> After stop" + closeProcess() } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 23672e4a42..2189ef39d1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import java.nio.file.FileSystems import java.nio.file.NoSuchFileException import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap +import java.util.function.Function import com.google.common.hash.HashCode import groovy.transform.PackageScope @@ -38,6 +39,7 @@ import nextflow.script.BodyDef import nextflow.script.ScriptType import nextflow.script.TaskClosure import nextflow.script.bundle.ResourcesBundle +import nextflow.script.params.CmdEvalParam import nextflow.script.params.EnvInParam import nextflow.script.params.EnvOutParam import nextflow.script.params.FileInParam @@ -447,10 +449,16 @@ class TaskRun implements Cloneable { /** * Look at the {@code nextflow.script.FileOutParam} which name is the expected * output name - * */ List getOutputFilesNames() { - cache0.computeIfAbsent('outputFileNames', (it)-> getOutputFilesNames0()) + // note: use an explicit function instead of a closure or lambda syntax, otherwise + // when calling this method from a subclass it will result into a MissingMethodExeception + // see https://issues.apache.org/jira/browse/GROOVY-2433 + cache0.computeIfAbsent('outputFileNames', new Function>() { + @Override + List apply(String s) { + return getOutputFilesNames0() + }}) } private List getOutputFilesNames0() { @@ -587,11 +595,40 @@ class TaskRun implements Cloneable { List getOutputEnvNames() { final items = getOutputsByType(EnvOutParam) - return items ? new ArrayList(items.keySet()*.name) : Collections.emptyList() + if( !items ) + return List.of() + final result = new ArrayList(items.size()) + for( EnvOutParam it : items.keySet() ) { + if( !it.name ) throw new IllegalStateException("Missing output environment name - offending parameter: $it") + result.add(it.name) + } + return result + } + + /** + * @return A {@link Map} instance holding a collection of key-pairs + * where the key represents a environment variable name holding the command + * output and the value the command the executed. + */ + Map getOutputEvals() { + final items = getOutputsByType(CmdEvalParam) + final result = new LinkedHashMap(items.size()) + for( CmdEvalParam it : items.keySet() ) { + if( !it.name ) throw new IllegalStateException("Missing output eval name - offending parameter: $it") + result.put(it.name, it.getTarget(context)) + } + return result } Path getCondaEnv() { - cache0.computeIfAbsent('condaEnv', (it)-> getCondaEnv0()) + // note: use an explicit function instead of a closure or lambda syntax, otherwise + // when calling this method from a subclass it will result into a MissingMethodExeception + // see https://issues.apache.org/jira/browse/GROOVY-2433 + cache0.computeIfAbsent('condaEnv', new Function() { + @Override + Path apply(String it) { + return getCondaEnv0() + }}) } private Path getCondaEnv0() { @@ -603,7 +640,14 @@ class TaskRun implements Cloneable { } Path getSpackEnv() { - cache0.computeIfAbsent('spackEnv', (it)-> getSpackEnv0()) + // note: use an explicit function instead of a closure or lambda syntax, otherwise + // when calling this method from a subclass it will result into a MissingMethodExeception + // see https://issues.apache.org/jira/browse/GROOVY-2433 + cache0.computeIfAbsent('spackEnv', new Function() { + @Override + Path apply(String it) { + return getSpackEnv0() + }}) } private Path getSpackEnv0() { @@ -617,7 +661,14 @@ class TaskRun implements Cloneable { } protected ContainerInfo containerInfo() { - cache0.computeIfAbsent('containerInfo', (it)-> containerInfo0()) + // note: use an explicit function instead of a closure or lambda syntax, otherwise + // when calling this method from a subclass it will result into a MissingMethodExeception + // see https://issues.apache.org/jira/browse/GROOVY-2433 + cache0.computeIfAbsent('containerInfo', new Function() { + @Override + ContainerInfo apply(String s) { + return containerInfo0() + }}) } private ContainerInfo containerInfo0() { diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskStartParams.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskStartParams.groovy index 41f22bf376..11f9a6ead2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskStartParams.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskStartParams.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskStatus.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskStatus.groovy index 97ac63efb2..cb2288e0f2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskStatus.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskStatus.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskTemplateEngine.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskTemplateEngine.groovy index 4d80825aed..e8372f8c7a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskTemplateEngine.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskTemplateEngine.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/tip/DefaultTaskTipProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/tip/DefaultTaskTipProvider.groovy new file mode 100644 index 0000000000..ee5be5d07d --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/processor/tip/DefaultTaskTipProvider.groovy @@ -0,0 +1,47 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.processor.tip + +import groovy.transform.CompileStatic +import nextflow.processor.TaskRun + +/** + * Implements the default tip provider + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class DefaultTaskTipProvider implements TaskTipProvider { + + static final private List TIPS = [ + 'when you have fixed the problem you can continue the execution adding the option `-resume` to the run command line', + "you can try to figure out what's wrong by changing to the process work dir and showing the script file named `${TaskRun.CMD_SCRIPT}`" as String, + "view the complete command output by changing to the process work dir and entering the command `cat ${TaskRun.CMD_OUTFILE}`" as String, + "you can replicate the issue by changing to the process work dir and entering the command `bash ${TaskRun.CMD_RUN}`" as String + ] + + static final private Random rnd = new Random() + + boolean enabled() { return true } + + @Override + String suggestTip(List context) { + return TIPS[ rnd.nextInt( TIPS.size() ) ] + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/tip/TaskTipProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/tip/TaskTipProvider.groovy new file mode 100644 index 0000000000..21654e5a1a --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/processor/tip/TaskTipProvider.groovy @@ -0,0 +1,33 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.processor.tip + +import org.pf4j.ExtensionPoint + +/** + * Basic interface for tips suggestion + * + * @author Paolo Di Tommaso + */ +interface TaskTipProvider extends ExtensionPoint { + + boolean enabled() + + String suggestTip(List context) + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy index f7c0a10da4..dd304ad59e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -242,6 +242,7 @@ class AssetManager { * @param name A project name or URL e.g. {@code cbcrg/foo} or {@code https://github.com/cbcrg/foo.git} * @return The fully qualified project name e.g. {@code cbcrg/foo} */ + @PackageScope String resolveName( String name ) { assert name @@ -362,11 +363,6 @@ class AssetManager { return this } - AssetManager setForce( boolean value ) { - this.force = value - return this - } - AssetManager checkValidRemoteRepo(String revision=null) { // Configure the git provider to use the required revision as source for all needed remote resources: // - config if present in repo (nextflow.config by default) @@ -801,7 +797,7 @@ class AssetManager { result.current = current // current branch name result.master = master // master branch name result.branches = branches // collection of branches - result.tags = tags // collect of tags + result.tags = tags // collect of tags return result } @@ -913,7 +909,7 @@ class AssetManager { def current = getCurrentRevision() if( current != defaultBranch ) { if( !revision ) { - throw new AbortOperationException("Project `$project` is currently stickied on revision: $current -- you need to explicitly specify a revision with the option `-r` in order to use it") + throw new AbortOperationException("Project `$project` is currently stuck on revision: $current -- you need to explicitly specify a revision with the option `-r` in order to use it") } } if( !revision || revision == current ) { @@ -1086,7 +1082,7 @@ class AssetManager { protected String guessHubProviderFromGitConfig(boolean failFast=false) { assert localPath - + // find the repository remote URL from the git project config file final domain = getGitConfigRemoteDomain() if( !domain && failFast ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy index 8c8a500bfa..a8cb617bf9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AzureRepositoryProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy index 2e0392b21d..4d5d50dcd1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketRepositoryProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy index de80f039b5..d7627bf6e5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/BitbucketServerRepositoryProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/GitUrl.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/GitUrl.groovy index f32114068c..d95d45a895 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/GitUrl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/GitUrl.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy index a7b6435d4e..55c71a74db 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/GithubRepositoryProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy index d03123333a..51a787d45c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/GitlabRepositoryProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/LocalRepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/LocalRepositoryProvider.groovy index 07aa21ab74..c30d145751 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/LocalRepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/LocalRepositoryProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/ProviderConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/ProviderConfig.groovy index 9a58ac1354..63a891ac4b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/ProviderConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/ProviderConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/ProviderPath.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/ProviderPath.groovy index 8e4815cf09..8d73d965fd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/ProviderPath.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/ProviderPath.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy index bd4ce8e478..0856ce24da 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy index 4c83a22e1f..96a959c569 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy index 58498fec71..5ec823b5a8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScript.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +20,12 @@ import java.lang.reflect.InvocationTargetException import java.nio.file.Paths import groovy.util.logging.Slf4j +import nextflow.NF import nextflow.NextflowMeta import nextflow.Session import nextflow.exception.AbortOperationException +import nextflow.secret.SecretsLoader + /** * Any user defined script will extends this class, it provides the base execution context * @@ -39,6 +42,8 @@ abstract class BaseScript extends Script implements ExecutionContext { private WorkflowDef entryFlow + private OutputDef publisher + @Lazy InputStream stdin = { System.in }() BaseScript() { @@ -55,6 +60,10 @@ abstract class BaseScript extends Script implements ExecutionContext { (ScriptBinding)super.getBinding() } + Session getSession() { + session + } + /** * Holds the configuration object which will used to execution the user tasks */ @@ -84,8 +93,9 @@ abstract class BaseScript extends Script implements ExecutionContext { binding.setVariable( 'workDir', session.workDir ) binding.setVariable( 'workflow', session.workflowMetadata ) binding.setVariable( 'nextflow', NextflowMeta.instance ) - binding.setVariable('launchDir', Paths.get('./').toRealPath()) - binding.setVariable('moduleDir', meta.moduleDir ) + binding.setVariable( 'launchDir', Paths.get('./').toRealPath() ) + binding.setVariable( 'moduleDir', meta.moduleDir ) + binding.setVariable( 'secrets', SecretsLoader.secretContext() ) } protected process( String name, Closure body ) { @@ -113,6 +123,17 @@ abstract class BaseScript extends Script implements ExecutionContext { meta.addDefinition(workflow) } + protected output(Closure closure) { + if( !NF.outputDefinitionEnabled ) + throw new IllegalStateException("Workflow output definition requires the `nextflow.preview.output` feature flag") + if( !entryFlow ) + throw new IllegalStateException("Workflow output definition must be defined after the anonymous workflow") + if( ExecutionStack.withinWorkflow() ) + throw new IllegalStateException("Workflow output definition is not allowed within a workflow") + + publisher = new OutputDef(closure) + } + protected IncludeDef include( IncludeDef include ) { if(ExecutionStack.withinWorkflow()) throw new IllegalStateException("Include statement is not allowed within a workflow definition") @@ -152,7 +173,7 @@ abstract class BaseScript extends Script implements ExecutionContext { if( !entryFlow ) { if( meta.getLocalWorkflowNames() ) - log.warn "No entry workflow specified" + throw new AbortOperationException("No entry workflow specified") if( meta.getLocalProcessNames() ) { final msg = """\ ============================================================================= @@ -175,6 +196,8 @@ abstract class BaseScript extends Script implements ExecutionContext { // invoke the entry workflow session.notifyBeforeWorkflowExecution() final ret = entryFlow.invoke_a(BaseScriptConsts.EMPTY_ARGS) + if( publisher ) + publisher.run(session.publishTargets) session.notifyAfterWorkflowExecution() return ret } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BaseScriptConsts.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BaseScriptConsts.groovy index 5710ed3c3c..6e69964e9f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BaseScriptConsts.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BaseScriptConsts.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,5 +25,5 @@ class BaseScriptConsts { public static Object[] EMPTY_ARGS = [] as Object[] - public static List PRIVATE_NAMES = ['session','processFactory','taskProcessor','meta','entryFlow'] + public static List PRIVATE_NAMES = ['session','processFactory','taskProcessor','meta','entryFlow', 'publisher'] } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BindableDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BindableDef.groovy index e1c51c02e6..e9b9fc502c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BindableDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BindableDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/BodyDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/BodyDef.groovy index 15161e5bce..0b55d8799e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/BodyDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/BodyDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ChainableDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ChainableDef.groovy index 650b029648..dbecb78725 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ChainableDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ChainableDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy index 85beb3c65a..bb18518bb9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ChannelOut.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ComponentDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ComponentDef.groovy index f3ab254e2b..2c0dd67eaf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ComponentDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ComponentDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/CompositeDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/CompositeDef.groovy index f857713517..bca0374744 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/CompositeDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/CompositeDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ExecutionContext.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ExecutionContext.groovy index 35e9dda9c2..75d3cea74f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ExecutionContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ExecutionContext.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ExecutionStack.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ExecutionStack.groovy index dececa1b45..5c2e62f8b4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ExecutionStack.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ExecutionStack.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/FunctionDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/FunctionDef.groovy index 102fbff1ff..3df4a67159 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/FunctionDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/FunctionDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/FusionMetadata.groovy b/modules/nextflow/src/main/groovy/nextflow/script/FusionMetadata.groovy new file mode 100644 index 0000000000..6786eb6323 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/script/FusionMetadata.groovy @@ -0,0 +1,49 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.script + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import groovy.util.logging.Slf4j +import nextflow.fusion.FusionConfig +import nextflow.Session +/** + * Models Fusion metadata for Nextflow execution + * + * @author Marco De La Pierre + */ +@Slf4j +@CompileStatic +@ToString(includeNames = true, includePackage = false) +@EqualsAndHashCode +class FusionMetadata { + boolean enabled + String version + + FusionMetadata(Session session) { + final FusionConfig fusionConfig = FusionConfig.getConfig(session) + this.enabled = fusionConfig.enabled() + this.version = fusionConfig.version() + } + + FusionMetadata(Boolean enabled, String version) { + this.enabled = enabled + this.version = version + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy index 3dd0ea1aad..08e6e5566e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/IncludeDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/IterableDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/IterableDef.groovy index baff26a993..6a578b8806 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/IterableDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/IterableDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/OutputDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/OutputDef.groovy new file mode 100644 index 0000000000..73de6e16ee --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/script/OutputDef.groovy @@ -0,0 +1,47 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.script + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel +/** + * Models the workflow output definition + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class OutputDef { + + private Closure closure + + OutputDef(Closure closure) { + this.closure = closure + } + + void run(Map targets) { + final dsl = new OutputDsl() + final cl = (Closure)closure.clone() + cl.setDelegate(dsl) + cl.setResolveStrategy(Closure.DELEGATE_FIRST) + cl.call() + + dsl.build(targets) + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy new file mode 100644 index 0000000000..d7bb9c4f09 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/script/OutputDsl.groovy @@ -0,0 +1,260 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.script + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.exception.ScriptRuntimeException +import nextflow.extension.CH +import nextflow.extension.MixOp +import nextflow.extension.PublishOp +import nextflow.file.FileHelper +/** + * Implements the DSL for publishing workflow outputs + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class OutputDsl { + + private Map publishConfigs = [:] + + private Path directory + + private Map defaults = [:] + + private volatile List ops = [] + + void directory(String directory) { + if( this.directory ) + throw new ScriptRuntimeException("Publish directory cannot be defined more than once in the workflow publish definition") + this.directory = FileHelper.toCanonicalPath(directory) + } + + void contentType(String value) { + setDefault('contentType', value) + } + + void contentType(boolean value) { + setDefault('contentType', value) + } + + void ignoreErrors(boolean value) { + setDefault('ignoreErrors', value) + } + + void mode(String value) { + setDefault('mode', value) + } + + void overwrite(boolean value) { + setDefault('overwrite', value) + } + + void overwrite(String value) { + setDefault('overwrite', value) + } + + void storageClass(String value) { + setDefault('storageClass', value) + } + + void tags(Map value) { + setDefault('tags', value) + } + + void enabled( boolean value ) { + setDefault('enabled', value) + } + + private void setDefault(String name, Object value) { + if( defaults.containsKey(name) ) + throw new ScriptRuntimeException("Default `${name}` option cannot be defined more than once in the workflow publish definition") + defaults[name] = value + } + + void target(String name, Closure closure) { + if( publishConfigs.containsKey(name) ) + throw new ScriptRuntimeException("Target '${name}' is defined more than once in the workflow publish definition") + + final dsl = new TargetDsl() + final cl = (Closure)closure.clone() + cl.setResolveStrategy(Closure.DELEGATE_FIRST) + cl.setDelegate(dsl) + cl.call() + + publishConfigs[name] = dsl.getOptions() + } + + void build(Map targets) { + // construct mapping of target name -> source channels + final Map> publishSources = [:] + for( final source : targets.keySet() ) { + final name = targets[source] + if( !name ) + continue + if( name !in publishSources ) + publishSources[name] = [] + publishSources[name] << source + } + + // create publish op (and optional index op) for each target + for( final name : publishSources.keySet() ) { + final sources = publishSources[name] + final mixed = sources.size() > 1 + ? new MixOp(sources.collect( ch -> CH.getReadChannel(ch) )).apply() + : sources.first() + final opts = publishOptions(name, publishConfigs[name] ?: [:]) + + ops << new PublishOp(CH.getReadChannel(mixed), opts).apply() + } + } + + private Map publishOptions(String name, Map overrides) { + if( !directory ) + directory = FileHelper.toCanonicalPath('.') + + final opts = defaults + overrides + if( opts.containsKey('ignoreErrors') ) + opts.failOnError = !opts.remove('ignoreErrors') + if( !opts.containsKey('overwrite') ) + opts.overwrite = 'standard' + + final path = opts.path as String ?: name + if( path.startsWith('/') ) + throw new ScriptRuntimeException("Invalid publish target path '${path}' -- it should be a relative path") + opts.path = directory.resolve(path) + + if( opts.index && !(opts.index as Map).path ) + throw new ScriptRuntimeException("Index file definition for publish target '${name}' is missing `path` option") + + return opts + } + + boolean getComplete() { + for( final op : ops ) + if( !op.complete ) + return false + return true + } + + static class TargetDsl { + + private Map opts = [:] + + void contentType(String value) { + setOption('contentType', value) + } + + void contentType(boolean value) { + setOption('contentType', value) + } + + void ignoreErrors(boolean value) { + setOption('ignoreErrors', value) + } + + void index(Closure closure) { + final dsl = new IndexDsl() + final cl = (Closure)closure.clone() + cl.setResolveStrategy(Closure.DELEGATE_FIRST) + cl.setDelegate(dsl) + cl.call() + setOption('index', dsl.getOptions()) + } + + void mode(String value) { + setOption('mode', value) + } + + void overwrite(boolean value) { + setOption('overwrite', value) + } + + void overwrite(String value) { + setOption('overwrite', value) + } + + void path(String value) { + setOption('path', value) + } + + void storageClass(String value) { + setOption('storageClass', value) + } + + void tags(Map value) { + setOption('tags', value) + } + + void enabled( boolean value ) { + setOption('enabled', value) + } + + private void setOption(String name, Object value) { + if( opts.containsKey(name) ) + throw new ScriptRuntimeException("Publish option `${name}` cannot be defined more than once for a given target") + opts[name] = value + } + + Map getOptions() { + return opts + } + + } + + static class IndexDsl { + + private Map opts = [:] + + void header(boolean value) { + setOption('header', value) + } + + void header(List value) { + setOption('header', value) + } + + void mapper(Closure value) { + setOption('mapper', value) + } + + void path(String value) { + setOption('path', value) + } + + void sep(String value) { + setOption('sep', value) + } + + private void setOption(String name, Object value) { + if( opts.containsKey(name) ) + throw new ScriptRuntimeException("Index option `${name}` cannot be defined more than once for a given index definition") + opts[name] = value + } + + Map getOptions() { + return opts + } + + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy index a9c2a86cf9..a17e80eb5a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,13 @@ package nextflow.script +import static nextflow.util.CacheHelper.* + import java.util.regex.Pattern import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.Const -import nextflow.NF import nextflow.ast.NextflowDSLImpl import nextflow.exception.ConfigParseException import nextflow.exception.IllegalConfigException @@ -30,8 +31,24 @@ import nextflow.executor.BashWrapperBuilder import nextflow.processor.ConfigList import nextflow.processor.ErrorStrategy import nextflow.processor.TaskConfig -import static nextflow.util.CacheHelper.HashMode -import nextflow.script.params.* +import nextflow.script.params.CmdEvalParam +import nextflow.script.params.DefaultInParam +import nextflow.script.params.DefaultOutParam +import nextflow.script.params.EachInParam +import nextflow.script.params.EnvInParam +import nextflow.script.params.EnvOutParam +import nextflow.script.params.FileInParam +import nextflow.script.params.FileOutParam +import nextflow.script.params.InParam +import nextflow.script.params.InputsList +import nextflow.script.params.OutParam +import nextflow.script.params.OutputsList +import nextflow.script.params.StdInParam +import nextflow.script.params.StdOutParam +import nextflow.script.params.TupleInParam +import nextflow.script.params.TupleOutParam +import nextflow.script.params.ValueInParam +import nextflow.script.params.ValueOutParam /** * Holds the process configuration properties @@ -47,14 +64,15 @@ class ProcessConfig implements Map, Cloneable { 'accelerator', 'afterScript', 'arch', + 'array', 'beforeScript', 'cache', + 'cleanup', + 'clusterOptions', 'conda', - 'cpus', 'container', 'containerOptions', - 'cleanup', - 'clusterOptions', + 'cpus', 'debug', 'disk', 'echo', // deprecated @@ -62,9 +80,8 @@ class ProcessConfig implements Map, Cloneable { 'executor', 'ext', 'fair', - 'machineType', - 'queue', 'label', + 'machineType', 'maxSubmitAwait', 'maxErrors', 'maxForks', @@ -74,9 +91,15 @@ class ProcessConfig implements Map, Cloneable { 'penv', 'pod', 'publishDir', + 'queue', + 'resourceLabels', + 'resourceLimits', 'scratch', + 'secret', 'shell', 'spack', + 'stageInMode', + 'stageOutMode', 'storeDir', 'tag', 'time', @@ -85,12 +108,8 @@ class ProcessConfig implements Map, Cloneable { 'val', 'each', 'env', - 'secret', 'stdin', 'stdout', - 'stageInMode', - 'stageOutMode', - 'resourceLabels' ] /** @@ -145,6 +164,11 @@ class ProcessConfig implements Map, Cloneable { */ private outputs = new OutputsList() + /** + * Map of default publish targets + */ + private Map publishTargets = [:] + /** * Initialize the taskConfig object with the defaults values * @@ -495,6 +519,13 @@ class ProcessConfig implements Map, Cloneable { outputs } + /** + * Typed shortcut to {@code #publishTargets} + */ + Map getPublishTargets() { + publishTargets + } + /** * Implements the process {@code debug} directive. */ @@ -572,6 +603,15 @@ class ProcessConfig implements Map, Cloneable { .bind(obj) } + OutParam _out_eval(Object obj ) { + new CmdEvalParam(this).bind(obj) + } + + OutParam _out_eval(Map opts, Object obj ) { + new CmdEvalParam(this) + .setOptions(opts) + .bind(obj) + } OutParam _out_file( Object obj ) { // note: check that is a String type to avoid to force @@ -623,6 +663,13 @@ class ProcessConfig implements Map, Cloneable { result } + void _publish_target(String emit, String name) { + final emitNames = outputs.collect { param -> param.channelEmitName } + if( emit !in emitNames ) + throw new IllegalArgumentException("Invalid emit name '${emit}' in publish statement, valid emits are: ${emitNames.join(', ')}") + publishTargets[emit] = name + } + /** * Defines a special *dummy* input parameter, when no inputs are * provided by the user for the current task @@ -977,4 +1024,34 @@ class ProcessConfig implements Map, Cloneable { return this } + int getArray() { + final value = configProperties.get('array') + if( value == null ) + return 0 + if( value instanceof Closure ) + throw new IllegalArgumentException("Process directive `array` cannot be declared in a dynamic manner with a closure") + try { + final result = value as Integer + if( result < 0 ) + throw new IllegalArgumentException("Process directive `array` cannot be a negative number") + if( result == 1 ) + throw new IllegalArgumentException("Process directive `array` should be greater than 1") + return result + } + catch( NumberFormatException e ) { + throw new IllegalArgumentException("Process directive `array` should be an integer greater than 1 -- offending value: '$value'", e) + } + } + + private static final List VALID_RESOURCE_LIMITS = List.of('cpus', 'memory', 'disk', 'time') + + ProcessConfig resourceLimits( Map entries ) { + for( entry in entries ) + if( entry.key !in VALID_RESOURCE_LIMITS ) + throw new IllegalArgumentException("Not a valid directive in `resourceLimits`: $entry.key") + + configProperties.put('resourceLimits', entries) + return this + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy index caac95d11d..f7e59b371e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessDef.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package nextflow.script import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Const import nextflow.Global import nextflow.Session @@ -206,7 +207,15 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef { } // make a copy of the output list because execution can change it - final copyOuts = declaredOutputs.clone() + output = new ChannelOut(declaredOutputs.clone()) + + // register process publish targets + for( final entry : processConfig.getPublishTargets() ) { + final emit = entry.key + final name = entry.value + final source = (DataflowWriteChannel)output.getProperty(emit) + session.publishTargets[source] = name + } // create the executor final executor = session @@ -221,7 +230,7 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef { // the result channels assert declaredOutputs.size()>0, "Process output should contains at least one channel" - return output = new ChannelOut(copyOuts) + return output } } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy index 56dd23838a..c4b776f318 100755 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessFactory.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy index 610209a47c..e844ab82c4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptBinding.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import groovy.util.logging.Slf4j import nextflow.NF import nextflow.Session import nextflow.exception.AbortOperationException -import nextflow.secret.SecretHolder import org.apache.commons.lang.StringUtils /** * Defines the script execution context. By default provided the following variables @@ -277,11 +276,7 @@ class ScriptBinding extends WorkflowBinding { } private String put0(String name, Object value) { - if( value instanceof SecretHolder ) { - log.warn "`params.$name` cannot be assigned to a secret value -- Assignment is ignored" - return null - } - + // keep track of the real name realNames << name diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptFile.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptFile.groovy index 83f775914d..1dadb029c6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptFile.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptFile.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy index 784b40d784..17a42f6f91 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy index e491124003..8931cf9b03 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptParser.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy index ea9bae557f..1cbbc68024 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import nextflow.Global import nextflow.Session import nextflow.exception.AbortOperationException import nextflow.exception.AbortRunException +import nextflow.plugin.Plugins import nextflow.util.HistoryFile /** * Run a nextflow script file @@ -257,6 +258,7 @@ class ScriptRunner { protected shutdown() { session.destroy() + Plugins.stop() session.cleanup() Global.cleanUp() log.debug "> Execution complete -- Goodbye" diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy index c7d38a1b7c..72498b2452 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptTokens.groovy @@ -1,5 +1,5 @@ /* - * Copyright 2013-2023, Seqera Labs + * Copyright 2013-2024, Seqera Labs * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -109,6 +109,19 @@ class TokenEnvCall { Object val } +/** + * Token used by the DSL to identify a command output declaration, like this + * 

+ *     input:
+ *     tuple( eval(X), ... )
+ *     
+ */
+@ToString
+@EqualsAndHashCode
+@TupleConstructor
+class TokenEvalCall {
+    Object val
+}
 
 /**
  * This class is used to identify a 'val' when used like in this example:
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ScriptType.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ScriptType.groovy
index 0faae5a9d5..890c5d59de 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/ScriptType.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/ScriptType.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/TaskClosure.java b/modules/nextflow/src/main/groovy/nextflow/script/TaskClosure.java
index 7d964aa15b..279f916382 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/TaskClosure.java
+++ b/modules/nextflow/src/main/groovy/nextflow/script/TaskClosure.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WaveMetadata.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WaveMetadata.groovy
new file mode 100644
index 0000000000..39692eb196
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WaveMetadata.groovy
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.script
+
+import groovy.transform.CompileStatic
+import groovy.transform.EqualsAndHashCode
+import groovy.transform.ToString
+import groovy.util.logging.Slf4j
+import nextflow.Session
+/**
+ * Models Wave metadata for Nextflow execution
+ * 
+ * @author Marco De La Pierre 
+ */
+@Slf4j
+@CompileStatic
+@ToString(includeNames = true, includePackage = false)
+@EqualsAndHashCode
+class WaveMetadata {
+
+    final boolean enabled
+
+    WaveMetadata(Session session) {
+        this( session.config.wave as Map ?: Map.of() )
+    }
+
+    WaveMetadata(Map opts) {
+        this.enabled = opts.enabled as boolean
+    }
+
+    WaveMetadata(Boolean enabled) {
+        this.enabled = enabled
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowBinding.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowBinding.groovy
index 30c633422d..105d67a246 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowBinding.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowBinding.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ package nextflow.script
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
+import groovyx.gpars.dataflow.DataflowWriteChannel
 import nextflow.NF
 import nextflow.exception.IllegalInvocationException
 import nextflow.extension.OpCall
@@ -155,4 +156,13 @@ class WorkflowBinding extends Binding  {
         }
     }
 
+    void _publish_target(DataflowWriteChannel source, String name) {
+        owner.session.publishTargets[source] = name
+    }
+
+    void _publish_target(ChannelOut out, String name) {
+        for( final ch : out )
+            _publish_target(ch, name)
+    }
+
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
index ec54d3043a..7a74cc4fff 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowDef.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,7 +57,7 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
         this.name = name
         // invoke the body resolving in/out params
         final copy = (Closure)rawBody.clone()
-        final resolver = new WorkflowParamsResolver()
+        final resolver = new WorkflowParamsDsl()
         copy.setResolveStrategy(Closure.DELEGATE_FIRST)
         copy.setDelegate(resolver)
         this.body = copy.call()
@@ -199,7 +199,7 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
         collectInputs(binding, args)
         // invoke the workflow execution
         final closure = body.closure
-        closure.delegate = binding
+        closure.setDelegate(binding)
         closure.setResolveStrategy(Closure.DELEGATE_FIRST)
         closure.call()
         // collect the workflow outputs
@@ -210,11 +210,11 @@ class WorkflowDef extends BindableDef implements ChainableDef, IterableDef, Exec
 }
 
 /**
- * Hold workflow parameters
+ * Implements the DSL for defining workflow takes and emits
  */
 @Slf4j
 @CompileStatic
-class WorkflowParamsResolver {
+class WorkflowParamsDsl {
 
     static final private String TAKE_PREFIX = '_take_'
     static final private String EMIT_PREFIX = '_emit_'
@@ -234,23 +234,4 @@ class WorkflowParamsResolver {
         else
             throw new MissingMethodException(name, WorkflowDef, args)
     }
-
-    private Map argsToMap(Object args) {
-        if( args && args.getClass().isArray() ) {
-            if( ((Object[])args)[0] instanceof Map ) {
-                def map = (Map)((Object[])args)[0]
-                return new HashMap(map)
-            }
-        }
-        Collections.emptyMap()
-    }
-
-    private Map argToPublishOpts(Object args) {
-        final opts = argsToMap(args)
-        if( opts.containsKey('saveAs')) {
-            log.warn "Workflow publish does not support `saveAs` option"
-            opts.remove('saveAs')
-        }
-        return opts
-    }
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
index 22dc01f1b3..9a8fe29421 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowMetadata.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -189,11 +189,29 @@ class WorkflowMetadata {
      */
     boolean stubRun
 
+    /**
+     * Returns ``true`` whenever the current instance is in preview mode
+     */
+    boolean preview
+
     /**
      * Which container engine was used to execute the workflow
      */
     String containerEngine
 
+    /**
+     * Metadata specific to Wave, including:
+     * 
  • enabled: whether Wave is enabled
+     */
+    WaveMetadata wave
+
+    /**
+     * Metadata specific to Fusion, including:
+     * 
  • enabled: whether Fusion is enabled
+     * 
  • version: the version of Fusion in use
+     */
+    FusionMetadata fusion
+
     /**
      * The list of files that concurred to create the config object
      */
@@ -240,6 +258,7 @@ class WorkflowMetadata {
         this.sessionId = session.uniqueId
         this.resume = session.resumeMode
         this.stubRun = session.stubRun
+        this.preview = session.preview
         this.runName = session.runName
         this.containerEngine = containerEngine0(session)
         this.configFiles = session.configFiles?.collect { it.toAbsolutePath() }
@@ -247,6 +266,8 @@ class WorkflowMetadata {
         this.userName = System.getProperty('user.name')
         this.homeDir = Paths.get(System.getProperty('user.home'))
         this.manifest = session.getManifest()
+        this.wave = new WaveMetadata(session)
+        this.fusion = new FusionMetadata(session)
 
         // check if there's a onComplete action in the config file
         registerConfigAction(session.config.workflow as Map)
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy
index 13228a2b42..364a3af1e8 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy b/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy
index 456369a317..93e0994c57 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/bundle/ResourcesBundle.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
index efe613805b..a7ccdb1d6e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy
index 8cf9408175..c811fa06fc 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseOutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy
index 4a5165f8df..cd470b35fd 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/BaseParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/CmdEvalParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/CmdEvalParam.groovy
new file mode 100644
index 0000000000..786f3fbaea
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/CmdEvalParam.groovy
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.script.params
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import groovy.transform.InheritConstructors
+import groovy.transform.Memoized
+
+/**
+ * Model process `output: eval PARAM` definition
+ *
+ * @author Paolo Di Tommaso 
+ */
+@InheritConstructors
+class CmdEvalParam extends BaseOutParam implements OptionalParam {
+
+    private static AtomicInteger counter = new AtomicInteger()
+
+    private Object target
+
+    private int count
+
+    {
+        count = counter.incrementAndGet()
+    }
+
+    String getName() {
+        return "nxf_out_eval_${count}"
+    }
+
+    BaseOutParam bind( def obj ) {
+        if( obj !instanceof CharSequence )
+            throw new IllegalArgumentException("Invalid argument for command output: $this")
+        // the target value object
+        target = obj
+        return this
+    }
+
+    @Memoized
+    String getTarget(Map context) {
+        return target instanceof GString
+            ? target.cloneAsLazy(context).toString()
+            : target.toString()
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy
index 80b85ad311..d17b49c303 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy
index ab0995f9b2..1857a46cb7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/DefaultOutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
index 2404fca43d..5313a16e94 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/EachInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy
index f5d784a878..2ca7e121d6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/EnvInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy
index 812c739b90..70086a89ba 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/EnvOutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -41,6 +41,12 @@ class EnvOutParam extends BaseOutParam implements OptionalParam {
         if( obj instanceof TokenVar ) {
             this.nameObj = obj.name
         }
+        else if( obj instanceof CharSequence ) {
+            this.nameObj = obj.toString()
+        }
+        else {
+            throw new IllegalArgumentException("Unexpected environment output definition - it should be either a string or a variable identifier - offending value: ${obj?.getClass()?.getName()}")
+        }
 
         return this
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
index 397a759b15..fb93eb1701 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/FileInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
index 97721ea7b2..6e6badeabd 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/FileOutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy
index d41e3c5c28..0551ddbe3b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/InParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy
index c389fe4e80..2aee0d671d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/InputsList.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy
index b8385c27b9..fb04bfd620 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/MissingParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy
index 326483063f..cda12e8250 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/OptionalParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy
index e145beec56..907b2fe944 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/OutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy
index 9509c1ecec..a50a48a7c5 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/OutputsList.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy
index c9d57b0047..9f19f40137 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/PathQualifier.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy
index c465c1ba46..80a0fe22ed 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/StdInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy
index cb76414c00..34cf93ded6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/StdOutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy
index 6255954eec..d58a97f925 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/TupleInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 package nextflow.script.params
 
 import groovy.transform.InheritConstructors
-import nextflow.NF
+import nextflow.script.TokenEvalCall
 import nextflow.script.TokenEnvCall
 import nextflow.script.TokenFileCall
 import nextflow.script.TokenPathCall
@@ -76,6 +76,9 @@ class TupleInParam extends BaseInParam {
             else if( item instanceof TokenEnvCall ) {
                 newItem(EnvInParam).bind(item.val)
             }
+            else if( item instanceof TokenEvalCall ) {
+                throw new IllegalArgumentException('Command input declaration is not supported')
+            }
             else if( item instanceof TokenStdinCall ) {
                 newItem(StdInParam)
             }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy
index e0de9e24ed..91df753b8c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/TupleOutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 package nextflow.script.params
 
 import groovy.transform.InheritConstructors
-import nextflow.NF
+import nextflow.script.TokenEvalCall
 import nextflow.script.TokenEnvCall
 import nextflow.script.TokenFileCall
 import nextflow.script.TokenPathCall
@@ -59,6 +59,9 @@ class TupleOutParam extends BaseOutParam implements OptionalParam {
             else if( item instanceof TokenEnvCall ) {
                 create(EnvOutParam).bind(item.val)
             }
+            else if( item instanceof TokenEvalCall ) {
+                create(CmdEvalParam).bind(item.val)
+            }
             else if( item instanceof GString ) {
                 throw new IllegalArgumentException("Unqualified output path declaration is not allowed - replace `tuple \"$item\",..` with `tuple path(\"$item\"),..`")
             }
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy
index a4c53d8f9c..dad42ea27a 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/ValueInParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy b/modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy
index 944212087a..20d427d864 100644
--- a/modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/script/params/ValueOutParam.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/secret/LocalSecretsProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/secret/LocalSecretsProvider.groovy
index bb34a2a097..c567006e46 100644
--- a/modules/nextflow/src/main/groovy/nextflow/secret/LocalSecretsProvider.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/secret/LocalSecretsProvider.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2021, Sage-Bionetworks
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -49,7 +49,7 @@ class LocalSecretsProvider implements SecretsProvider, Closeable {
 
     private Map env = SysEnv.get()
 
-    private Map secretsMap
+    private Map secretsMap
 
     private Path storeFile
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/secret/NullProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/secret/NullProvider.groovy
new file mode 100644
index 0000000000..dbd244f43d
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/secret/NullProvider.groovy
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.secret
+
+import groovy.transform.CompileStatic
+import groovy.transform.PackageScope
+import nextflow.exception.AbortOperationException
+import org.eclipse.jgit.errors.NotSupportedException
+
+/**
+ * A secret provider only used to report an error when secrets are disabled
+ *
+ * @author Paolo Di Tommaso 
+ */
+@PackageScope
+@CompileStatic
+class NullProvider implements SecretsProvider {
+    @Override
+    boolean activable() {
+        return false
+    }
+
+    @Override
+    SecretsProvider load() {
+        return this
+    }
+
+    @Override
+    Secret getSecret(String name) {
+        throw new AbortOperationException("Unable to access 'secrets.$name' because secrets feature is disabled - Enable it setting the variable NXF_ENABLE_SECRETS=true in your environment")
+    }
+
+    @Override
+    void putSecret(String name, String value) {
+        throw new NotSupportedException("Operation 'putSecret' is not supported by ${this.class.name}")
+    }
+
+    @Override
+    void removeSecret(String name) {
+        throw new NotSupportedException("Operation 'removeSecret' is not supported by ${this.class.name}")
+    }
+
+    @Override
+    Set listSecretsNames() {
+        throw new NotSupportedException("Operation 'listSecretsNames' is not supported by ${this.class.name}")
+    }
+
+    @Override
+    String getSecretsEnv(List secretNames) {
+        throw new NotSupportedException("Operation 'getSecretsEnv' is not supported by ${this.class.name}")
+    }
+
+    @Override
+    String getSecretsEnv() {
+        throw new NotSupportedException("Operation 'getSecretsEnv' is not supported by ${this.class.name}")
+    }
+
+    @Override
+    void close() throws IOException {
+
+    }
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/secret/SecretHolder.groovy b/modules/nextflow/src/main/groovy/nextflow/secret/SecretHolder.groovy
deleted file mode 100644
index 23efa0bbab..0000000000
--- a/modules/nextflow/src/main/groovy/nextflow/secret/SecretHolder.groovy
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2021, Sage-Bionetworks
- *
- * All Rights reserved
- *
- */
-
-package nextflow.secret
-
-/**
- * Hold a config secret
- *
- * @author Paolo Di Tommaso 
- */
-class SecretHolder extends Closure {
-
-    private String secretName
-
-    private SecretsProvider provider
-
-    SecretHolder(String name) {
-        super(null, null);
-        setResolveStrategy(Closure.TO_SELF)
-        this.secretName = name
-    }
-
-    SecretHolder bind(SecretsProvider provider) {
-        this.provider = provider
-        return this
-    }
-
-    boolean isBound() {
-        return provider != null
-    }
-
-    String getSecretName() {
-        return secretName
-    }
-
-    Object getSecretValue() {
-        if( provider == null )
-            throw new IllegalStateException("Missing secret provider — unable to resolve secret '$secretName'")
-        final secret = provider.getSecret(secretName)
-        if( secret==null )
-            throw new MissingSecretException("Unknown config secret '$secretName'")
-        return secret.value
-    }
-
-    Object call(final Object... args) {
-        return resolve0()
-    }
-
-    @Override
-    Object call() {
-        return resolve0()
-    }
-
-    private Object resolve0() {
-        isBound() ? getSecretValue() : "secrets.$secretName"
-    }
-}
diff --git a/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy b/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy
index 49d0f38ca0..2da21446b3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy
@@ -17,7 +17,6 @@
 
 package nextflow.secret
 
-import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
 import nextflow.SysEnv
 import nextflow.exception.AbortOperationException
@@ -32,12 +31,23 @@ import nextflow.plugin.Plugins
 @Singleton
 class SecretsLoader {
 
+    private SecretsProvider provider
+
     static boolean isEnabled() {
         SysEnv.get('NXF_ENABLE_SECRETS', 'true') == 'true'
     }
 
-    @Memoized
     SecretsProvider load() {
+        if( provider )
+            return provider
+        synchronized (this) {
+            if( provider )
+                return provider
+            return provider = load0()
+        }
+    }
+
+    private SecretsProvider load0() {
         // discover all available secrets provider
         final all = Plugins.getPriorityExtensions(SecretsProvider)
         // find first activable in the current environment
@@ -49,4 +59,24 @@ class SecretsLoader {
             throw new AbortOperationException("Unable to load secrets provider")
     }
 
+    void reset() {
+        provider=null
+    }
+
+    static protected makeSecretsContext(SecretsProvider provider) {
+
+        return new Object() {
+            def getProperty(String name) {
+                if( !provider )
+                    throw new AbortOperationException("Unable to resolve secrets.$name - no secret provider is available")
+                provider.getSecret(name)?.value
+            }
+        }
+    }
+
+    static Object secretContext() {
+        final provider = isEnabled() ? getInstance().load() : new NullProvider()
+        return makeSecretsContext(provider)
+    }
+    
 }
diff --git a/modules/nextflow/src/main/groovy/nextflow/sort/BigSort.java b/modules/nextflow/src/main/groovy/nextflow/sort/BigSort.java
index 934114c727..1c92bfbe12 100644
--- a/modules/nextflow/src/main/groovy/nextflow/sort/BigSort.java
+++ b/modules/nextflow/src/main/groovy/nextflow/sort/BigSort.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/sort/LevelDbSort.java b/modules/nextflow/src/main/groovy/nextflow/sort/LevelDbSort.java
index c7b8e9e389..f8e0006d37 100644
--- a/modules/nextflow/src/main/groovy/nextflow/sort/LevelDbSort.java
+++ b/modules/nextflow/src/main/groovy/nextflow/sort/LevelDbSort.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractBinarySplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractBinarySplitter.groovy
index 083f6f32bb..52910f9555 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractBinarySplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractBinarySplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy
index 728bbba4ec..09320afa59 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractTextSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractTextSplitter.groovy
index e9fd1b04b6..f50055139d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractTextSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractTextSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/BytesSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/BytesSplitter.groovy
index 727d3f6b80..c5df5bd40e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/BytesSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/BytesSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/CacheableCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/CacheableCollector.groovy
index 2889b409fc..1bb41f13ab 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/CacheableCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/CacheableCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/CharSequenceCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/CharSequenceCollector.groovy
index 5433930c70..3e6c1a5fbc 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/CharSequenceCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/CharSequenceCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/CollectorStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/CollectorStrategy.groovy
index 9d28c252a8..78f4b9590e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/CollectorStrategy.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/CollectorStrategy.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/CsvSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/CsvSplitter.groovy
index 6e6a6cb7e3..5b9fd73590 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/CsvSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/CsvSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/EntryCounter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/EntryCounter.groovy
index ad9bf9bdf2..36d3d8fd77 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/EntryCounter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/EntryCounter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/FastaSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/FastaSplitter.groovy
index fc492e55a0..76b6dbd75b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/FastaSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/FastaSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy
index 1a75c3dd17..8d8e43d50f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/HeaderCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/HeaderCollector.groovy
index cf18c99387..a9441d0907 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/HeaderCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/HeaderCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/JsonSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/JsonSplitter.groovy
index d68b2ed070..507cb94abc 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/JsonSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/JsonSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/ObjectListCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/ObjectListCollector.groovy
index 57661222ff..c332062181 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/ObjectListCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/ObjectListCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterEx.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterEx.groovy
index 28be7e56fc..2a057767d7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterEx.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterEx.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy
index 13bd9f3ad4..e7d1d13168 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterStrategy.groovy
index ffeaf107e3..828b40ebc2 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterStrategy.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterStrategy.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/StringSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/StringSplitter.groovy
index 20e8f24f88..4206076da0 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/StringSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/StringSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/TextFileCollector.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/TextFileCollector.groovy
index f0066b7255..78d4f9bdd7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/TextFileCollector.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/TextFileCollector.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/TextSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/TextSplitter.groovy
index 9c4f607b73..9d6d095a04 100644
--- a/modules/nextflow/src/main/groovy/nextflow/splitter/TextSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/splitter/TextSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/AnsiLogObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/AnsiLogObserver.groovy
index aaf96f3a44..ff05d8f949 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/AnsiLogObserver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/AnsiLogObserver.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 
 package nextflow.trace
 
+import java.util.regex.Pattern
+
 import groovy.transform.CompileStatic
 import jline.TerminalFactory
 import nextflow.Session
@@ -85,6 +87,8 @@ class AnsiLogObserver implements TraceObserver {
 
     private volatile int cols = 80
 
+    private volatile int rows = 24
+
     private long startTimestamp
 
     private long endTimestamp
@@ -116,7 +120,7 @@ class AnsiLogObserver implements TraceObserver {
     }
 
     synchronized void appendInfo(String message) {
-        if( message==null )
+        if( message==null || message.isEmpty() )
             return
         boolean warn
         if( isHashLogPrefix(message) && !(warn=message.indexOf('NOTE:')>0) )
@@ -181,7 +185,7 @@ class AnsiLogObserver implements TraceObserver {
                 wait(200)
             }
         }
-        // 
+        //
         final stats = statsObserver.getStats()
         renderProgress(stats)
         renderSummary(stats)
@@ -227,7 +231,7 @@ class AnsiLogObserver implements TraceObserver {
     protected String getExecutorName(String key) {
         session.getExecutorFactory().getDisplayName(key)
     }
-    
+
     protected void renderExecutors(Ansi term) {
         int count=0
         def line = ''
@@ -237,7 +241,7 @@ class AnsiLogObserver implements TraceObserver {
         }
 
         if( count ) {
-            term.a("executor > " + line)
+            term.a(Attribute.INTENSITY_FAINT).a("executor > " + line).reset()
             term.newline()
         }
     }
@@ -251,6 +255,7 @@ class AnsiLogObserver implements TraceObserver {
         }
 
         cols = TerminalFactory.get().getWidth()
+        rows = TerminalFactory.get().getHeight()
 
         // calc max width
         final now = System.currentTimeMillis()
@@ -265,9 +270,25 @@ class AnsiLogObserver implements TraceObserver {
             lastWidthReset = now
 
         // render line
+        def renderedLines = 0
+        def skippedLines = 0
         for( ProgressRecord entry : processes ) {
-            term.a(line(entry))
-            term.newline()
+            // Only show line if we have space in the visible terminal area
+            // or if the process has some submitted tasks
+            if( renderedLines <= rows - 5 || entry.getTotalCount() > 0 ) {
+                term.a(line(entry))
+                term.newline()
+                renderedLines += 1
+            }
+            // Process with no active tasks and we're out of screen space, skip
+            else {
+                skippedLines += 1
+            }
+        }
+        // Tell the user how many processes without active tasks were hidden
+        if( skippedLines > 0 ){
+            term.a(Attribute.ITALIC).a(Attribute.INTENSITY_FAINT).a("Plus ").bold().a(skippedLines).reset()
+            term.a(Attribute.ITALIC).a(Attribute.INTENSITY_FAINT).a(" more processes waiting for tasks…").reset().newline()
         }
         rendered = true
     }
@@ -322,7 +343,7 @@ class AnsiLogObserver implements TraceObserver {
             return
         if( enableSummary == null && delta <= 60*1_000 )
             return
-        
+
         if( session.isSuccess() && stats.progressLength>0 ) {
             def report = ""
             report += "Completed at: ${new Date(endTimestamp).format('dd-MMM-yyyy HH:mm:ss')}\n"
@@ -350,13 +371,13 @@ class AnsiLogObserver implements TraceObserver {
         if( color ) fmt = fmt.fg(Color.DEFAULT)
         AnsiConsole.out.println(fmt.eraseLine())
     }
-    
+
     protected void printAnsiLines(String lines) {
         final text = lines
                 .replace('\r','')
                 .replace(NEWLINE, ansi().eraseLine().toString() + NEWLINE)
         AnsiConsole.out.print(text)
-    } 
+    }
 
     protected String fmtWidth(String name, int width, int cols) {
         assert name.size() <= width
@@ -369,36 +390,103 @@ class AnsiLogObserver implements TraceObserver {
     }
 
     protected String fmtChop(String str, int cols) {
+        // Truncate the process name to fit the terminal width
         if( str.size() <= cols )
             return str
-        return cols>3 ? str[0..(cols-3-1)] + '...' : str[0..cols-1]
+        // Take the first 3 characters and the final chunk of text
+        //   eg. for: NFCORE_RNASEQ:RNASEQ:FASTQ_SUBSAMPLE_FQ_SALMON:FQ_SUBSAMPLE
+        //   truncate to: NFC…_SALMON:FQ_SUBSAMPLE
+        return cols>5 ? str.take(3) + '…' + str.takeRight(cols-1-3) : str[0..cols-1]
     }
 
-    protected String line(ProgressRecord stats) {
+    private final static Pattern TAG_REGEX = ~/ \((.+)\)( *)$/
+    private final static Pattern LBL_REPLACE = ~/ \(.+\) *$/
+
+    protected Ansi line(ProgressRecord stats) {
+        final term = ansi()
         final float tot = stats.getTotalCount()
         final float com = stats.getCompletedCount()
+        // Truncate or pad the label to the correct width
         final label = fmtWidth(stats.taskName, labelWidth, Math.max(cols-50, 5))
+        // Break up the process label into components for styling. eg:
+        //   NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:GUNZIP_GTF (genes.gtf.gz)
+        //     labelTag = genes.gtf.gz
+        //     labelSpaces = whitespace padding after process name
+        //     labelFinalProcess = GUNZIP_GTF
+        //     labelNoFinalProcess = NFCORE_RNASEQ:RNASEQ:PREPARE_GENOME:
+        final tagMatch = TAG_REGEX.matcher(label)
+        final labelTag = tagMatch ? tagMatch.group(1) : ''
+        final labelSpaces = tagMatch ? tagMatch.group(2) : ''
+        final labelNoTag = LBL_REPLACE.matcher(label).replaceFirst("")
+        final labelFinalProcess = labelNoTag.tokenize(':')[-1]
+        final labelNoFinalProcess = labelNoTag.dropRight(labelFinalProcess.length())
         final hh = (stats.hash && tot>0 ? stats.hash : '-').padRight(9)
 
-        if( tot == 0  )
-            return "[$hh] process > $label -"
-
         final x = tot ? Math.floor(com / tot * 100f).toInteger() : 0
-        final pct = "[${String.valueOf(x).padLeft(3)}%]".toString()
+        // eg. 100% (whitespace padded for alignment)
+        final pct = "${String.valueOf(x).padLeft(3)}%".toString()
+        // eg. 1 of 1
+        final numbs = " ${(int)com} of ${(int)tot}".toString()
+
+        // Task hash, eg: [fa/71091a]
+        term.a(Attribute.INTENSITY_FAINT).a('[').reset()
+        term.fg(Color.BLUE).a(hh).reset()
+        term.a(Attribute.INTENSITY_FAINT).a('] ').reset()
+
+        // Only show 'process > ' if the terminal has lots of width
+        if( cols > 180 )
+            term.a(Attribute.INTENSITY_FAINT).a('process > ').reset()
+        // Stem of process name, dim text
+        term.a(Attribute.INTENSITY_FAINT).a(labelNoFinalProcess).reset()
+        // Final process name, regular text
+        term.a(labelFinalProcess)
+        // Active process with a tag, eg: (genes.gtf.gz)
+        if( labelTag ){
+            // Tag in yellow, () dim but tag text regular
+            term.fg(Color.YELLOW).a(Attribute.INTENSITY_FAINT).a(' (').reset()
+            term.fg(Color.YELLOW).a(labelTag)
+            term.a(Attribute.INTENSITY_FAINT).a(')').reset().a(labelSpaces)
+        }
+
+        // No tasks
+        if( tot == 0 ) {
+            term.a(' -')
+            return term
+        }
 
-        final numbs = "${(int)com} of ${(int)tot}".toString()
-        def result = "[${hh}] process > $label $pct $numbs"
+        // Progress percentage, eg: [ 80%]
+        if( cols > 120 ) {
+            // Only show the percentage if we have lots of width
+            // Percentage text in green if 100%, otherwise blue
+            term.a(Attribute.INTENSITY_FAINT).a(' [').reset()
+                .fg(pct == '100%' ? Color.GREEN : Color.BLUE).a(pct).reset()
+                .a(Attribute.INTENSITY_FAINT).a(']').reset()
+        }
+        else {
+            // If narrow terminal, show single pipe char instead of percentage to save space
+            term.a(Attribute.INTENSITY_FAINT).a(' |').reset()
+        }
+        // Progress active task count, eg: 8 of 10
+        term.a(numbs)
+
+        // Completed task counts and status
+        // Dim text for cached, otherwise regular
         if( stats.cached )
-            result += ", cached: $stats.cached"
+            term.a(Attribute.INTENSITY_FAINT).a(", cached: $stats.cached").reset()
         if( stats.stored )
-            result += ", stored: $stats.stored"
+            term.a(", stored: $stats.stored")
         if( stats.failed )
-            result += ", failed: $stats.failed"
+            term.a(", failed: $stats.failed")
         if( stats.retries )
-            result += ", retries: $stats.retries"
-        if( stats.terminated && tot )
-            result += stats.errored ? ' \u2718' : ' \u2714'
-        return fmtChop(result, cols)
+            term.a(", retries: $stats.retries")
+        // Show red cross ('✘') or green tick ('✔') according to status
+        if( stats.terminated && tot ) {
+            if( stats.errored )
+                term.fg(Color.RED).a(' \u2718').reset()
+            else
+                term.fg(Color.GREEN).a(' \u2714').reset()
+        }
+        return term
     }
 
     @Override
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/GraphObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/GraphObserver.groovy
index ccadaf9597..9fc99f8f3d 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/GraphObserver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/GraphObserver.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/ProgressRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/ProgressRecord.groovy
index 513f2f4d3c..0e208f5736 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/ProgressRecord.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/ProgressRecord.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/ProgressState.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/ProgressState.groovy
index 0ffdf796f3..88a575ae7b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/ProgressState.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/ProgressState.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/ReportObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/ReportObserver.groovy
index b3886ac205..cf00fd0e37 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/ReportObserver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/ReportObserver.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/ReportSummary.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/ReportSummary.groovy
index b205e43cdc..f11c23a8f3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/ReportSummary.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/ReportSummary.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TimelineObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TimelineObserver.groovy
index 8ae3369613..93fd93a584 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/TimelineObserver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/TimelineObserver.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceFileObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceFileObserver.groovy
index 1ea090c9c9..2a37920e41 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceFileObserver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceFileObserver.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceHelper.groovy
index 62d5bff213..4c307118d4 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceHelper.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy
index c5fac42d00..636ad03287 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceObserver.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy
index 34f2072751..33c4a5d510 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStats.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStats.groovy
index 90cfbfec6c..6fe97c2431 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStats.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStats.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStatsObserver.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStatsObserver.groovy
index 0ad0464f93..d0f7c4e7fb 100644
--- a/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStatsObserver.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStatsObserver.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy
index bc76928b0c..406239c32e 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/Barrier.groovy b/modules/nextflow/src/main/groovy/nextflow/util/Barrier.groovy
index 8836a77160..408f6342e6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/Barrier.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/Barrier.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/BlankSeparatedList.groovy b/modules/nextflow/src/main/groovy/nextflow/util/BlankSeparatedList.groovy
index afdfef9b07..81ff619ead 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/BlankSeparatedList.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/BlankSeparatedList.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/BlockingBlockingQueue.groovy b/modules/nextflow/src/main/groovy/nextflow/util/BlockingBlockingQueue.groovy
index ce1be12ba4..b27e0d6e4f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/BlockingBlockingQueue.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/BlockingBlockingQueue.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/BlockingThreadExecutorFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/util/BlockingThreadExecutorFactory.groovy
index 44d03347da..925bfb6f51 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/BlockingThreadExecutorFactory.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/BlockingThreadExecutorFactory.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ClientProxyThrottler.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ClientProxyThrottler.groovy
index 5f06058619..b34c59197c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ClientProxyThrottler.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ClientProxyThrottler.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ClusterConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ClusterConfig.groovy
index 1611d4af1b..cbdf3db866 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ClusterConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ClusterConfig.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy
index f70490fdbe..f128585a71 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ConfigHelper.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@ import java.nio.file.Path
 import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
-import nextflow.secret.SecretHolder
 import org.codehaus.groovy.runtime.InvokerHelper
 /**
  * Helper method to handle configuration object
@@ -240,9 +239,7 @@ class ConfigHelper {
             return render0(val)
         if( val instanceof Map )
             return render0(val)
-        if( val instanceof SecretHolder )
-            return val.call()
-        
+
         InvokerHelper.inspect(val)
     }
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/CsvWriter.groovy b/modules/nextflow/src/main/groovy/nextflow/util/CsvWriter.groovy
new file mode 100644
index 0000000000..b2d4736bbc
--- /dev/null
+++ b/modules/nextflow/src/main/groovy/nextflow/util/CsvWriter.groovy
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2024, Ben Sherman
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.util
+
+import java.nio.file.Path
+
+import groovy.transform.CompileStatic
+
+@CompileStatic
+class CsvWriter {
+
+    private /* boolean | List */ header = false
+
+    private String sep = ','
+
+    CsvWriter(Map opts) {
+        if( opts.header )
+            this.header = opts.header
+
+        if( opts.sep )
+            this.sep = opts.sep.toString()
+    }
+
+    void apply(List records, Path path) {
+        Collection columns
+        if( header == true ) {
+            final first = records.first()
+            if( first !instanceof Map )
+                throw new IllegalArgumentException('Records must be map objects when header=true')
+            columns = ((Map)first).keySet()
+        }
+        else if( header instanceof List ) {
+            columns = header
+        }
+
+        path.delete()
+
+        if( columns )
+            path << columns.collect(it -> '"' + it + '"').join(sep) << '\n'
+
+        for( final record : records ) {
+            Collection values
+            if( record instanceof List )
+                values = record
+            else if( record instanceof Map )
+                values = columns
+                    ? record.subMap(columns).values()
+                    : record.values()
+            else
+                throw new IllegalArgumentException('Records must be list or map objects')
+
+            path << values.collect(it -> '"' + it + '"').join(sep) << '\n'
+        }
+    }
+
+}
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/CustomPoolFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/util/CustomPoolFactory.groovy
index b4cd4304d9..5a9788e85f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/CustomPoolFactory.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/CustomPoolFactory.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadFactory.groovy
index 5de2e6c7a4..87302a8d11 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadFactory.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadFactory.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadPool.java b/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadPool.java
index 365df327a8..43319503e7 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadPool.java
+++ b/modules/nextflow/src/main/groovy/nextflow/util/CustomThreadPool.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/HexIdentity.groovy b/modules/nextflow/src/main/groovy/nextflow/util/HexIdentity.groovy
index 675fb5922d..7eaa721e12 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/HexIdentity.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/HexIdentity.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/HistoryFile.groovy b/modules/nextflow/src/main/groovy/nextflow/util/HistoryFile.groovy
index f94eb59566..15d5cb83ca 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/HistoryFile.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/HistoryFile.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,9 +22,10 @@ import java.text.DateFormat
 import java.text.SimpleDateFormat
 
 import groovy.transform.EqualsAndHashCode
-import groovy.transform.Memoized
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
+import nextflow.Const
+import nextflow.SysEnv
 import nextflow.exception.AbortOperationException
 /**
  * Manages the history file containing the last 1000 executed commands
@@ -34,13 +35,12 @@ import nextflow.exception.AbortOperationException
 @Slf4j
 class HistoryFile extends File {
 
-    public static final String FILE_NAME = '.nextflow/history'
+    static String defaultFileName() { Const.appCacheDir.resolve('history').toString() }
 
     @Lazy
     public static final HistoryFile DEFAULT = { def f=new HistoryFile(); f.parentFile?.mkdirs(); return f } ()
 
-    @Memoized
-    static final disabled() { System.getenv('NXF_IGNORE_RESUME_HISTORY')=='true' }
+    static final disabled() { SysEnv.get('NXF_IGNORE_RESUME_HISTORY')=='true' }
 
     private static final DateFormat TIMESTAMP_FMT = new SimpleDateFormat('yyyy-MM-dd HH:mm:ss')
 
@@ -50,7 +50,7 @@ class HistoryFile extends File {
     private static final VAL_9 = (int)('9' as char)
 
     private HistoryFile() {
-        super(FILE_NAME)
+        super(defaultFileName())
     }
 
     HistoryFile(File file) {
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LockManager.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LockManager.groovy
index 4bb691c6bc..b23887b3b9 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/LockManager.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/LockManager.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy
index 82e4e0ad7b..fc8bd68841 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -179,7 +179,7 @@ class LoggerHelper {
 
         // -- add the S3 uploader by default
         if( !containsClassName(debugConf,traceConf, 'nextflow.cloud.aws.nio') )
-            debugConf << S3_UPLOADER_CLASS
+            debugConf << 'nextflow.cloud.aws.nio'
         if( !containsClassName(debugConf,traceConf, 'io.seqera') )
             debugConf << 'io.seqera'
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/MustacheTemplateEngine.groovy b/modules/nextflow/src/main/groovy/nextflow/util/MustacheTemplateEngine.groovy
index ec37a21531..f68a6c9774 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/MustacheTemplateEngine.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/MustacheTemplateEngine.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/NameGenerator.groovy b/modules/nextflow/src/main/groovy/nextflow/util/NameGenerator.groovy
index 9ad2641cb8..8877622f0c 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/NameGenerator.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/NameGenerator.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -90,6 +90,7 @@ class NameGenerator {
             "lethal",
             "lonely",
             "loving",
+            "loquacious",
             "mad",
             "magical",
             "maniac",
@@ -331,6 +332,9 @@ class NameGenerator {
             // Douglas Engelbart gave the mother of all demos: https://en.wikipedia.org/wiki/Douglas_Engelbart
             "engelbart",
 
+            // Maurits Cornelis Escher - Dutch graphic artist who made woodcuts, lithographs, and mezzotints, many of which were inspired by mathematics. His work features mathematical objects and operations including impossible objects, explorations of infinity, reflection, symmetry, perspective, truncated and stellated polyhedra, hyperbolic geometry, and tessellations. https://en.wikipedia.org/wiki/M._C._Escher
+            "escher",
+
             // Euclid invented geometry. https://en.wikipedia.org/wiki/Euclid
             "euclid",
 
@@ -786,6 +790,9 @@ class NameGenerator {
             // James Watson - American molecular biologist, geneticist and zoologist, best known as one of the co-discoverers of the structure of DNA. https://en.wikipedia.org/wiki/James_Watson
             "watson",
 
+            // Alfred Wegener - German climatologist, geologist, geophysicist, meteorologist, and polar researcher. Best known for proposing the theory of continental drift, the precursor to plate tectonics. https://en.wikipedia.org/wiki/Alfred_Wegener
+            "wegener",
+        
             // Marlyn Wescoff - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Marlyn_Meltzer
             "wescoff",
 
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/PathEscapeAware.groovy b/modules/nextflow/src/main/groovy/nextflow/util/PathEscapeAware.groovy
index 58196ea587..0281ee41e9 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/PathEscapeAware.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/PathEscapeAware.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/PathSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/util/PathSplitter.groovy
index 4a212e1a55..34881dda59 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/PathSplitter.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/PathSplitter.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/PathTrie.groovy b/modules/nextflow/src/main/groovy/nextflow/util/PathTrie.groovy
index 96901cb02a..a7bfbca14b 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/PathTrie.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/PathTrie.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ProxyConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ProxyConfig.groovy
index c2114115c8..1f12e56a04 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ProxyConfig.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ProxyConfig.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy b/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy
index c767a71c63..59dcf359fa 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/RemoteSession.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/SerializationHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/SerializationHelper.groovy
index d6668fde2d..05bae4acfc 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/SerializationHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/SerializationHelper.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ServiceName.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ServiceName.groovy
index 0458aacc34..2371d7429f 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ServiceName.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ServiceName.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/SimpleAgent.groovy b/modules/nextflow/src/main/groovy/nextflow/util/SimpleAgent.groovy
index 3843a6889b..7b2bc12ee3 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/SimpleAgent.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/SimpleAgent.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/SimpleHttpClient.groovy b/modules/nextflow/src/main/groovy/nextflow/util/SimpleHttpClient.groovy
index f3c953e397..07ac305fa6 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/SimpleHttpClient.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/SimpleHttpClient.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  * Copyright 2018, University of Tübingen, Quantitative Biology Center (QBiC)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/SpuriousDeps.groovy b/modules/nextflow/src/main/groovy/nextflow/util/SpuriousDeps.groovy
index 6ffa7a2381..dedbab6f86 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/SpuriousDeps.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/SpuriousDeps.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolBuilder.groovy
index 8614fbd4fe..20298106df 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolBuilder.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolBuilder.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolHelper.groovy
index 5b2ffccbc8..12c736bd60 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolHelper.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolManager.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolManager.groovy
index f326900383..d1ea726f74 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolManager.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ThreadPoolManager.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,7 +50,9 @@ class ThreadPoolManager {
     private Boolean allowThreadTimeout
     private Duration maxAwait = DEFAULT_MAX_AWAIT
     private ExecutorService executorService
-    final private String name
+    private String name
+    private String waitMsg = "Waiting for file transfers to complete (%d files)"
+    private String exitMsg = "Exiting before file transfers were completed -- Some files may be lost"
 
     ThreadPoolManager(String name) {
         this.name = name
@@ -72,6 +74,12 @@ class ThreadPoolManager {
         return this
     }
 
+    ThreadPoolManager withShutdownMessage(String waitMsg, String exitMsg) {
+        this.waitMsg = waitMsg
+        this.exitMsg = exitMsg
+        return this
+    }
+
     ExecutorService create() {
         if( minThreads>maxThreads ) {
             log.debug("Thread pool '$name' minThreads ($minThreads) cannot be greater than maxThreads ($maxThreads) - Setting minThreads to $maxThreads")
@@ -116,9 +124,7 @@ class ThreadPoolManager {
         }
 
         executorService.shutdown()
-        // wait for ongoing file transfer to complete
-        final waitMsg = "Waiting for file transfers to complete (%d files)"
-        final exitMsg = "Exiting before FileTransfer thread pool complete -- Some files may be lost"
+        // wait for remaining threads to complete
         ThreadPoolHelper.await(executorService, maxAwait, waitMsg, exitMsg)
         log.debug "Thread pool '$name' shutdown completed (hard=$hard)"
     }
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ThrottlingExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ThrottlingExecutor.groovy
index ed6231f874..da0fcaa067 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/ThrottlingExecutor.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/ThrottlingExecutor.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/Trie.groovy b/modules/nextflow/src/main/groovy/nextflow/util/Trie.groovy
index bc0a0fbc96..ac2e93c9ca 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/Trie.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/Trie.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/TupleHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/TupleHelper.groovy
index 6f0a3b1216..b857f21547 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/TupleHelper.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/TupleHelper.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/groovy/nextflow/util/VirtualThreadPool.groovy b/modules/nextflow/src/main/groovy/nextflow/util/VirtualThreadPool.groovy
index e805fc369f..8eccf202f1 100644
--- a/modules/nextflow/src/main/groovy/nextflow/util/VirtualThreadPool.groovy
+++ b/modules/nextflow/src/main/groovy/nextflow/util/VirtualThreadPool.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/java/com/esotericsoftware/minlog/Log.java b/modules/nextflow/src/main/java/com/esotericsoftware/minlog/Log.java
index 62baeb07aa..e5bb3e514f 100644
--- a/modules/nextflow/src/main/java/com/esotericsoftware/minlog/Log.java
+++ b/modules/nextflow/src/main/java/com/esotericsoftware/minlog/Log.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/java/groovy/runtime/metaclass/CustomMetaClassCreationHandle.java b/modules/nextflow/src/main/java/groovy/runtime/metaclass/CustomMetaClassCreationHandle.java
index eb5b62c2a6..8a640138be 100644
--- a/modules/nextflow/src/main/java/groovy/runtime/metaclass/CustomMetaClassCreationHandle.java
+++ b/modules/nextflow/src/main/java/groovy/runtime/metaclass/CustomMetaClassCreationHandle.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/java/groovy/runtime/metaclass/ExtensionProvider.java b/modules/nextflow/src/main/java/groovy/runtime/metaclass/ExtensionProvider.java
index 411c76257c..2ea1880db6 100644
--- a/modules/nextflow/src/main/java/groovy/runtime/metaclass/ExtensionProvider.java
+++ b/modules/nextflow/src/main/java/groovy/runtime/metaclass/ExtensionProvider.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/java/groovy/runtime/metaclass/NextflowDelegatingMetaClass.java b/modules/nextflow/src/main/java/groovy/runtime/metaclass/NextflowDelegatingMetaClass.java
index b67fcdcbb6..dc1525e088 100644
--- a/modules/nextflow/src/main/java/groovy/runtime/metaclass/NextflowDelegatingMetaClass.java
+++ b/modules/nextflow/src/main/java/groovy/runtime/metaclass/NextflowDelegatingMetaClass.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/java/groovy/runtime/metaclass/NumberDelegatingMetaClass.java b/modules/nextflow/src/main/java/groovy/runtime/metaclass/NumberDelegatingMetaClass.java
index d1cd108d3f..86e8c149bd 100644
--- a/modules/nextflow/src/main/java/groovy/runtime/metaclass/NumberDelegatingMetaClass.java
+++ b/modules/nextflow/src/main/java/groovy/runtime/metaclass/NumberDelegatingMetaClass.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/resources/META-INF/build-info.properties b/modules/nextflow/src/main/resources/META-INF/build-info.properties
index f0b524e10b..64c35c2296 100644
--- a/modules/nextflow/src/main/resources/META-INF/build-info.properties
+++ b/modules/nextflow/src/main/resources/META-INF/build-info.properties
@@ -1,4 +1,4 @@
-build=5903
-version=24.01.0-edge
-timestamp=1707175051052
-commitId=8b9905c7f
+build=5913
+version=24.04.1
+timestamp=1716198535910
+commitId=2ee0c418c
diff --git a/modules/nextflow/src/main/resources/META-INF/extensions.idx b/modules/nextflow/src/main/resources/META-INF/extensions.idx
index 73ade8f597..81f00bd334 100644
--- a/modules/nextflow/src/main/resources/META-INF/extensions.idx
+++ b/modules/nextflow/src/main/resources/META-INF/extensions.idx
@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2023, Seqera Labs
+# Copyright 2013-2024, Seqera Labs
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,3 +23,4 @@ nextflow.container.resolver.DefaultContainerResolver
 nextflow.mail.SendMailProvider
 nextflow.mail.SimpleMailProvider
 nextflow.mail.JavaMailProvider
+nextflow.processor.tip.DefaultTaskTipProvider
diff --git a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt
index 2b9d5122d8..6f1cb315ff 100644
--- a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt
+++ b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt
@@ -1,9 +1,9 @@
-nf-amazon@2.4.0
-nf-azure@1.5.0
-nf-cloudcache@0.4.0
+nf-amazon@2.5.2
+nf-azure@1.6.0
+nf-cloudcache@0.4.1
 nf-codecommit@0.2.0
-nf-console@1.1.0
-nf-ga4gh@1.2.0
-nf-google@1.11.0
-nf-tower@1.8.0
-nf-wave@1.3.0
\ No newline at end of file
+nf-console@1.1.3
+nf-ga4gh@1.3.0
+nf-google@1.13.2
+nf-tower@1.9.1
+nf-wave@1.4.2
\ No newline at end of file
diff --git a/modules/nextflow/src/main/resources/META-INF/services/org.codehaus.groovy.runtime.ExtensionModule b/modules/nextflow/src/main/resources/META-INF/services/org.codehaus.groovy.runtime.ExtensionModule
index d553b440f3..632b0bf71c 100644
--- a/modules/nextflow/src/main/resources/META-INF/services/org.codehaus.groovy.runtime.ExtensionModule
+++ b/modules/nextflow/src/main/resources/META-INF/services/org.codehaus.groovy.runtime.ExtensionModule
@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2023, Seqera Labs
+# Copyright 2013-2024, Seqera Labs
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html b/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html
index 2c2a22cc7b..9bb91c48b0 100644
--- a/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html
+++ b/modules/nextflow/src/main/resources/nextflow/dag/mermaid.dag.template.html
@@ -1,5 +1,5 @@