Merge branch 'master' into 3372-validate-published-outputs

Signed-off-by: Ben Sherman <[email protected]>
nextflow-io · May 19, 2024 · f3eb942 · f3eb942
2 parents 787e053 + e2e6081
commit f3eb942
Show file tree

Hide file tree

Showing 1,237 changed files with 12,741 additions and 5,350 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,3 @@
+# DOCUMENTATION -------------------------------------------------------
+# Docs folder ownership
+/docs/ @nextflow-io/docs
diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md
@@ -4,7 +4,6 @@ When submitting a Pull Request please make sure to not include
 in the changeset any modification in these files:
 
 * `nextflow`
-* `docs/conf.py`
 * `modules/nf-commons/src/main/nextflow/Const.groovy`
 
 Also, please sign-off the DCO [1] to certify you are the author of the contribution

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -39,9 +39,17 @@ jobs:
 
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@v43
         with:
-          files_ignore: docs
+          files_ignore: docs/**
+
+      - name: List all changed files
+        env:
+            ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+            for file in ${ALL_CHANGED_FILES}; do
+                echo "$file was changed"
+            done
 
       - name: Setup env
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -54,7 +62,7 @@ jobs:
 
       - name: Setup Java ${{ matrix.java_version }}
         if: steps.changed-files.outputs.any_changed == 'true'
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{matrix.java_version}}
           distribution: 'temurin'
@@ -67,7 +75,14 @@ jobs:
 
       - name: Test
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: make test
+        run: |
+            # configure test env
+            if [[ "$GOOGLE_SECRET" ]]; then
+            echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
+            export GOOGLE_APPLICATION_CREDENTIALS=$PWD/google_credentials.json
+            fi
+            # run tests
+            make test
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -82,7 +97,7 @@ jobs:
           AZURE_BATCH_ACCOUNT_KEY: ${{ secrets.AZURE_BATCH_ACCOUNT_KEY }}
 
       - name: Publish tests report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: steps.changed-files.outputs.any_changed == 'true' && always()
         with:
           name: report-unit-tests-jdk-${{ matrix.java_version }}
@@ -96,6 +111,7 @@ jobs:
     if: ${{ !contains(github.event.head_commit.message, '[ci fast]') && needs.build.outputs.any_changed == 'true' }}
     needs: build
     runs-on: ubuntu-latest
+    timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
@@ -117,7 +133,7 @@ jobs:
           NXF_GITHUB_ACCESS_TOKEN: ${{ secrets.NXF_GITHUB_ACCESS_TOKEN }}
 
       - name: Setup Java ${{ matrix.java_version }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{matrix.java_version}}
           distribution: 'temurin'
@@ -152,7 +168,7 @@ jobs:
         run: tar -cvf integration-tests.tar tests/checks
 
       - name: Publish tests report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: report-${{ matrix.test_mode }}-jdk-${{ matrix.java_version }}

diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 #
-#  Copyright 2013-2023, Seqera Labs
+#  Copyright 2013-2024, Seqera Labs
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/NOTICE b/NOTICE
@@ -1,6 +1,6 @@
 NEXTFLOW
 
-Copyright 2013-2023, Seqera Labs
+Copyright 2013-2024, Seqera Labs
 
 
 This software includes source code and libraries developed by:
@@ -142,7 +142,7 @@ This software includes source code and libraries developed by:
 
     Spack
     https://github.com/spack/spack
-    Copyright 2013-2023 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details.
+    Copyright 2013-2024 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details.
     Licensed under Apache License, Version 2.0
 
 Nextflow is a registered trademark of Seqera Labs, Spain.

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-24.01.0-edge
+24.04.1
diff --git a/build.gradle b/build.gradle
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -69,6 +69,10 @@ allprojects {
     java {
         toolchain {
             languageVersion = JavaLanguageVersion.of(19)
+            // note: the use of Java 21 causes the error "NoClassDefFoundError: java/util/SequencedCollection"
+            // see also
+            //   https://aphyr.com/posts/369-classnotfoundexception-java-util-sequencedcollection
+            //   https://www.baeldung.com/java-21-sequenced-collections
         }
     }
 
@@ -108,15 +112,15 @@ allprojects {
         }
 
         // Documentation required libraries
-        groovyDoc 'org.fusesource.jansi:jansi:1.11'
-        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.18"
-        groovyDoc "org.apache.groovy:groovy-ant:4.0.18"
+        groovyDoc 'org.fusesource.jansi:jansi:2.4.0'
+        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.21"
+        groovyDoc "org.apache.groovy:groovy-ant:4.0.21"
     }
 
     test {
         useJUnitPlatform()
     }
-    
+
     // this is required due to this IDEA bug
     // https://youtrack.jetbrains.com/issue/IDEA-129282
     sourceSets {
@@ -145,28 +149,27 @@ allprojects {
     }
 
     // Required to run tests on Java 9 and higher in compatibility mode
-    if (JavaVersion.current() >= JavaVersion.VERSION_1_9) {
-        tasks.withType(Test) {
-            jvmArgs ([
-                    '--enable-preview',
-                    '--add-opens=java.base/java.lang=ALL-UNNAMED',
-                    '--add-opens=java.base/java.io=ALL-UNNAMED',
-                    '--add-opens=java.base/java.nio=ALL-UNNAMED',
-                    '--add-opens=java.base/java.nio.file.spi=ALL-UNNAMED',
-                    '--add-opens=java.base/java.net=ALL-UNNAMED',
-                    '--add-opens=java.base/java.util=ALL-UNNAMED',
-                    '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED',
-                    '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED',
-                    '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED',
-                    '--add-opens=java.base/sun.nio.fs=ALL-UNNAMED',
-                    '--add-opens=java.base/sun.net.www.protocol.http=ALL-UNNAMED',
-                    '--add-opens=java.base/sun.net.www.protocol.https=ALL-UNNAMED',
-                    '--add-opens=java.base/sun.net.www.protocol.ftp=ALL-UNNAMED',
-                    '--add-opens=java.base/sun.net.www.protocol.file=ALL-UNNAMED',
-                    '--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED',
-                    '--add-opens=java.base/jdk.internal.vm=ALL-UNNAMED',
-            ])
-        }
+    tasks.withType(Test) {
+        jvmArgs ([
+            '-Dorg.spockframework.mock.ignoreByteBuddy=true',
+            '--enable-preview',
+            '--add-opens=java.base/java.lang=ALL-UNNAMED',
+            '--add-opens=java.base/java.io=ALL-UNNAMED',
+            '--add-opens=java.base/java.nio=ALL-UNNAMED',
+            '--add-opens=java.base/java.nio.file.spi=ALL-UNNAMED',
+            '--add-opens=java.base/java.net=ALL-UNNAMED',
+            '--add-opens=java.base/java.util=ALL-UNNAMED',
+            '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED',
+            '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED',
+            '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED',
+            '--add-opens=java.base/sun.nio.fs=ALL-UNNAMED',
+            '--add-opens=java.base/sun.net.www.protocol.http=ALL-UNNAMED',
+            '--add-opens=java.base/sun.net.www.protocol.https=ALL-UNNAMED',
+            '--add-opens=java.base/sun.net.www.protocol.ftp=ALL-UNNAMED',
+            '--add-opens=java.base/sun.net.www.protocol.file=ALL-UNNAMED',
+            '--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED',
+            '--add-opens=java.base/jdk.internal.vm=ALL-UNNAMED',
+        ])
     }
 
     /**
@@ -213,21 +216,13 @@ task buildInfo { doLast {
     timestamp=${System.currentTimeMillis()}
     commitId=${project.property('commitId')}
     """.stripIndent()
-    
+
     // -- update 'nextflow' wrapper
     file0 = file('nextflow')
     src = file0.text
     src = src.replaceAll(/NXF_VER\=\$\{NXF_VER:-'.*'\}/, 'NXF_VER=\\${NXF_VER:-\'' + version + '\'}')
     file0.text = src
 
-    // -- update sphynx
-    def major = version.split(/\./)[0..1].join('.')
-    file0 = file('docs/conf.py')
-    src = file0.text
-    src = src.replaceAll(/version *= *'[0-9a-zA-Z_\-\.]+'/, "version = '$major'" as String)
-    src = src.replaceAll(/release *= *'[0-9a-zA-Z_\-\.]+'/, "release = '$version'" as String)
-    file0.text = src
-
     // -- update dockerfile
     file0 = file('docker/Dockerfile')
     src = file0.text

diff --git a/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy b/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/changelog.txt b/changelog.txt
@@ -1,5 +1,149 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.04.1 - 20 May 2024 
+- Fix nf-google plugin dependency [725e2860]
+- Fix nf-amazon plugin dependency [c234b09f]
+- Bump [email protected] [633989a6]
+- Bump [email protected] [e31f70e0]
+
+24.04.0 - 20 May 2024
+- Add 'preview' to workflow runtime metadata (#4985) [935bb1e5]
+- Add enabled property to output dsl (#5008) [284415b1]
+- Fix Taskbar API is not supported error [0ea09ccc]
+- Fix inspect should not write history entry [c713ad51]
+- Fix unexpected container resolution [a5ecf8a4]
+- Improve icon loading error handling [c72e16f8]
+- Remove Fusion symlink resolution (#5004) [071ea74c]
+- Remove `seqera` and `defaults` from Conda default channels (#5003) [ec5ebd0b]
+- Use protected visibility for updateStatus method [6871ba06]
+- Workflow output definition (#4784) [cf0546b1]
+- Bump Fusion 2.3 (#5005) [7176c113]
+- Bump groovy-console 4.0.21-patch.2 [eb97831f]
+- Bump [email protected] [96ee633d]
+- Bump [email protected] [e8359042]
+- Bump [email protected] [5dcb4c7a]
+- Bump [email protected] [73c668a6]
+
+24.04.0-edge - 13 May 2024
+- Add Wave and Fusion info to workflow metadata (#4945) [bb7e1c8e]
+- Add `k8s.cpuLimits` config option (#3027) [3c6e96d0]
+- Add account config option for grid executors (#4975) [a09e37dd]
+- Add git to docs deps [aa9e1273]
+- Add resourceLimits directive (#2911) [7c9d965e]
+- Add support for Job arrays (#3892) [ca9bc9d4]
+- Add support for clusterOptions as a list of values (#4993) [dd173e33]
+- Add threads dump for troubleshooting purposes [8992ebde]
+- Fix Gstring casting exception when clusterOptions is a closure [74004fbd]
+- Fix Missing error code when no entry is specified [f507e9a4]
+- Fix NPE in LogsCheckpoint class [deb3076d]
+- Fix Prevent maxForks less than zero [7676dd9c]
+- Fix Use fully qualified S3 uris in error message (#4923) [f1cffd1b]
+- Fix Wave container resolution with singularity and ociMode [54ad6241]
+- Fix collectFile saving to GCS with sort: false (#4965) [1418553a]
+- Fix console "Plugin manager not initialised" warn (#4989) [5ff44538]
+- Fix console icon (#4991) [b8a23706]
+- Fix docs formatting [c02e58c7]
+- Fix docs snippet [6499649d]
+- Fix flaky docs test (#4957) [ea8246f6]
+- Fix groovy console issue (#4988) [b9bf6410]
+- Fix job array docs (#4984) [6a3347ee]
+- Fix missing include error message (#4981) [aad100e1]
+- Fix script error text alignment (#4681) [1dc4e4e4]
+- Fix security vulnerability in logback (#4947) [0ffcc4ca]
+- Fix semaphore in parallel polling monitor (#4927) [5c37fcc2]
+- Fix remove commented out test lines in Azure Batch Pool opts tests. (#4914) [cb607f07]
+- Guarantees K8s pod name is unique on resume (#4959) [361cef84]
+- Improve config resolution docs (#4950) [019eb86c]
+- Improve documentation about azcopy installation requirements for custom Azure Batch worker pools (#4911) [5c410db8]
+- Improve error message for image pull time-out for Singularity/Apptainer/Charliecloud (#4974) [73015fbd]
+- Remove unused const [6e91285d]
+- Run task finalisation asynchronously (#4890) [e0e94227]
+- Strip auth secret from logs [acf63e0e]
+- Update TES executor to TES API v1.1 (#4195) [7b32c2d6]
+- Update aws.md to include Cluster access (#4951) [459d725b]
+- Update developer diagrams (#4922) [dcff41a5]
+- Update operator return types (#4976) [a614fbe7]
+- Update stale documentation in overview.md (#4968) [6a58c6d7]
+- Use for instead eachLine in error formatting [4a821f46]
+- azure batch autopool feature more comprehensive documentation (#4941) [adbd8903]
+- Bump Gradle 8.7 [8b5cf3cc]
+- Bump [email protected] [830b032c]
+- Bump [email protected] [163683c2]
+- Bump [email protected] [6d99a22a]
+- Bump [email protected] [89695ed3]
+- Bump [email protected] [357b143a]
+- Bump [email protected] [6c62a60a]
+
+24.03.0-edge - 15 Apr 2024
+- Add custom jobName for Google Batch [df40d55f]
+- Add escher to name generator class [2e6496e2]
+- Add retry policy to Google Batch client [c4981dcc]
+- Add retry strategy for publishing (#4839) [c9c7032c]
+- Add support for Azure custom startTask (#4913) [27d01e3a]
+- Add task tip extension point [eadad5b8]
+- Allow secrets to be used in pipeline script (#4171) [df866a24]
+- Do not print a new line when stdout is empty (#4892) [658a5ec8]
+- Fix Azure pool creation [2ee4d11e]
+- Fix Use of secrets in the includeConfig path [00c9f226]
+- Fix coloured ANSI log bug (#4898) [a04d6983]
+- Fix eval output type via bash -c wrapping (#4887) [2165a14d]
+- Fix exception handling in local executor [74d7d7a8]
+- Fix support for GCS requester pays bucket option [d9d61cff]
+- Fix test when missing Google secret [33dc3ce0]
+- Improve Charliecloud support (#4879) [287471c0]
+- Improve control on azcopy install (#4883) [01447d5c]
+- Improve error message when Google creds file is corrupted [a550e52f]
+- Improve getting started docs (#4764) [b59111b3]
+- Improve retry logic for AWS Batch executor [62926c28]
+- Nextflow launch script: improving search for JAVA_CMD (#4830) [ebbbe9e7]
+- Publish built-in reports as Tower reports (#4760) [b710d923]
+- Remove not needed dsl=2 + error in example (#4812) [7c5779d7]
+- Remove unused code from AssetManager [77365165]
+- Revert "Fix failing CI tests (#4861)" [7ba2e253]
+- Update NameGenerator (#4907) [248201af]
+- Update Platform API endpoint (#4855) [4842423a]
+- Update Wave to API v1alpha2 (#4906) [9c350872]
+- Update docs (#4852) [6e2d1a94]
+- Updated docs on Google Cloud setup and credentials (#4896) [7e8b5e26]
+- Updated dodcs note on singularity default command (#4825) [567f5334]
+- Bump groovy 4.0.20 [66c1a164]
+- Bump groovy 4.0.21 [9e08390b]
+- Bump [email protected] [fc70dc8c]
+- Bump [email protected] [b0c4e2c5]
+- Bump [email protected] [6ae25fad]
+- Bump [email protected] [967c2ac8]
+- Bump [email protected] [ddda969e]
+
+24.02.0-edge - 10 Mar 2024
+- Add K8s job ttlSecondsAfterFinished option (#4434) [93627be6]
+- Add NXF_CACHE_DIR environment var (#4655) [4b00170a]
+- Add colours to ansi logs  (#4573) [5e2ce9ed]
+- Add eval output type (#4493) [df978113]
+- Fix Always emit publish event for symlinks on resume (#4790) [bb5c4f9d]
+- Fix Do not create local plugin path in embedded mode [9d6dd6a0]
+- Fix Error while publishing S3 file with blanks [b74c0227]
+- Fix Missing dependency for console command [baf29110]
+- Fix typo in Azure Batch docs ('Azore') (#4735) [192bf8df]
+- Fix typo in error message [a7f23305]
+- Remove experimental admonition for podman [17d0dced]
+- Remove square brackets from job name in LSF executor (#4799) [6e0ac72d]
+- Remove unneeded const [09c957fb]
+- Rename Tower -> Seqera Platform in docs and log messages (#4727) [7caffef9]
+- Update Azure dependencies [1bcbaf0d]
+- Update copyright info [e3089f0e]
+- Use alias for HistoryFile.Record [17217a1c]
+- minor cli docstring fix (#4759) [ee4b4a25]
+- Bump Grengine 3.0.2 [42ca2b6f]
+- Bump groovy 4.0.19 [854dc1f0]
+- Bump snakeyaml 2.2 [07480779]
+- Bump [email protected] [0eb84071]
+- Bump [email protected] [d63be8c0]
+- Bump [email protected] [57b7004e]
+- Bump [email protected] [b7f703f5]
+- Bump [email protected] [b8ffb180]
+- Bump [email protected] [0c542eda]
+- Bump amazoncorretto 17.0.10-al2023 [3e695ad9]
+
 24.01.0-edge - 5 Feb 2024
 - Add support for custom fuse device plugin (#4612) [a1e33193]
 - Fix Ignore stored process message when ansi log is enabled (#4645) [f9ba47ef]
@@ -28,7 +172,7 @@ NEXTFLOW CHANGE-LOG
 - Bump [email protected] [2bc698c7]
 - Bump [email protected] [07415ce1]
 - Bump [email protected] [b991e14b]
-- Bump Groovy 4 (#4443) [ci fast] [9d32503b]
+- Bump Groovy 4 (#4443) [9d32503b]
 - Bump actions/checkout@v4 [d1b3195e]
 - Bump [email protected] + [email protected] [331ff425]
 - Bump nextflow 23.12.0-edge as min version [63e83702]