From 2275c03cce4c7cd097b1c0210a9deb603f5fa301 Mon Sep 17 00:00:00 2001 From: Doug Daniels Date: Fri, 23 Sep 2022 11:26:49 -0500 Subject: [PATCH 01/12] Apply GCP resourceLabels to the VirtualMachine (#3234) Signed-off-by: Doug Daniels Signed-off-by: Ben Sherman Co-authored-by: Ben Sherman --- .../cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy | 1 + .../google/lifesciences/GoogleLifeSciencesHelperTest.groovy | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy index 15acff8284..be08d069e3 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelper.groovy @@ -197,6 +197,7 @@ class GoogleLifeSciencesHelper { .setDisks([disk]) .setServiceAccount(serviceAccount) .setPreemptible(req.preemptible) + .setLabels(req.resourceLabels) def network = new Network() diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy index b5b9f2600c..24896e5683 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesHelperTest.groovy @@ -194,7 +194,8 @@ class GoogleLifeSciencesHelperTest extends GoogleSpecification { network: 'net/123', subnetwork: 'sub/192', serviceAccountEmail: 'myaccount@developer.gserviceaccount.com', - usePrivateAddress: true )) + usePrivateAddress: true, + resourceLabels: [foo:'bar'] )) then: with(resources3) { getVirtualMachine().getMachineType() == type @@ -208,6 +209,7 @@ class GoogleLifeSciencesHelperTest extends GoogleSpecification { getVirtualMachine().getAccelerators()[0].getType()=='nvidia-tesla-k80' getVirtualMachine().getBootDiskSizeGb() == 75 getVirtualMachine().getCpuPlatform() == 'Intel Skylake' + getVirtualMachine().getLabels() == [foo: 'bar'] getVirtualMachine().getNetwork().getUsePrivateAddress() getVirtualMachine().getNetwork().getNetwork() == 'net/123' getVirtualMachine().getNetwork().getSubnetwork() == 'sub/192' From 26f1f8962e4c83cb0ee2da2cb7c40b71690296e1 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 24 Sep 2022 10:12:27 +0200 Subject: [PATCH 02/12] Bump fusion 0.4.x [ci fast] Signed-off-by: Paolo Di Tommaso --- .../src/main/io/seqera/wave/plugin/config/FusionConfig.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/FusionConfig.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/FusionConfig.groovy index 263db50ea5..4c93caef34 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/FusionConfig.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/FusionConfig.groovy @@ -27,7 +27,7 @@ import groovy.transform.CompileStatic @CompileStatic class FusionConfig { - final static public String DEFAULT_FUSION_URL = 'https://fusionfs-release.s3.eu-west-1.amazonaws.com/fusionfs-latest.json' + final static public String DEFAULT_FUSION_URL = 'https://s3.eu-west-1.amazonaws.com/fusionfs.io/releases/v0.4.json' final private enabled final private String containerConfigUrl From 738422152dc2194cb9b0942c878f0253307054ae Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 25 Sep 2022 09:11:07 +0200 Subject: [PATCH 03/12] Improve wave error reporting Signed-off-by: Paolo Di Tommaso --- .../src/main/io/seqera/wave/plugin/WaveClient.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy index f860ab707a..5807669c9c 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy @@ -159,11 +159,13 @@ class WaveClient { try { final resp = httpClient.send(req, HttpResponse.BodyHandlers.ofString()) - if( resp.statusCode()==200 ) { - log.debug "Wave response: ${resp.body()}" + log.debug "Wave response: statusCode=${resp.statusCode()}; body=${resp.body()}" + if( resp.statusCode()==200 ) return jsonToSubmitResponse(resp.body()) - } - throw new BadResponseException("Wave invalid response: [${resp.statusCode()}] ${resp.body()}") + if( resp.statusCode()==401 ) + throw new BadResponseException("Unauthorised [401] - Verify you have provided a valid access token") + else + throw new BadResponseException("Wave invalid response: [${resp.statusCode()}] ${resp.body()}") } catch (ConnectException e) { throw new IllegalStateException("Unable to connect Wave service: $endpoint") From 9b9df15ded9e27bf01b93b7c333b75ba7f6e4a5f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 11:06:44 +0200 Subject: [PATCH 04/12] Add Wave endpoint Signed-off-by: Paolo Di Tommaso --- .../src/main/io/seqera/wave/plugin/config/WaveConfig.groovy | 2 +- .../src/test/io/seqera/wave/plugin/config/WaveConfigTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy index e2b51ae195..10b4e75cb8 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy @@ -28,7 +28,7 @@ import nextflow.util.Duration @Slf4j @CompileStatic class WaveConfig { - final private static String DEF_ENDPOINT = 'http://localhost:9090' + final private static String DEF_ENDPOINT = 'https://wave.seqera.io' final private Boolean enabled final private String endpoint final private List containerConfigUrl diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/WaveConfigTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/WaveConfigTest.groovy index 17aebf6f9b..c06fc7a5d1 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/WaveConfigTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/config/WaveConfigTest.groovy @@ -32,7 +32,7 @@ class WaveConfigTest extends Specification { def opts = new WaveConfig([:]) then: !opts.enabled() - opts.endpoint() == 'http://localhost:9090' + opts.endpoint() == 'https://wave.seqera.io' } def 'should create from env' () { From 5aec28ac77fdb9eeca16c13ba5cd75f8827adb68 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 11:07:24 +0200 Subject: [PATCH 05/12] Update Google Batch mount point with the requirements Fixes #3244 Signed-off-by: Sean Davis Signed-off-by: Paolo Di Tommaso --- .../batch/GoogleBatchScriptLauncher.groovy | 2 +- .../GoogleBatchScriptLauncherTest.groovy | 22 +++++++++---------- .../batch/GoogleBatchTaskHandlerTest.groovy | 8 +++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index 2a2629541a..bca544fa16 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -40,7 +40,7 @@ import nextflow.util.PathTrie @CompileStatic class GoogleBatchScriptLauncher extends BashWrapperBuilder { - private static final String MOUNT_ROOT = '/mnt' + private static final String MOUNT_ROOT = '/mnt/disks' private CloudStoragePath remoteWorkDir private Path remoteBinDir diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchScriptLauncherTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchScriptLauncherTest.groovy index a5543dfa73..62dfe74e04 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchScriptLauncherTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchScriptLauncherTest.groovy @@ -40,10 +40,10 @@ class GoogleBatchScriptLauncherTest extends Specification{ and: launcher.getContainerMounts() == [MOUNTS] where: - BUCKET | PATH | PARENT | EXPECTED | MOUNTS - 'foo' | '/' | false | Paths.get('/mnt/foo') | '/mnt/foo:/mnt/foo:rw' - 'foo' | '/some/dir' | false | Paths.get('/mnt/foo/some/dir') | '/mnt/foo/some/dir:/mnt/foo/some/dir:rw' - 'foo' | '/some/dir' | true | Paths.get('/mnt/foo/some/dir') | '/mnt/foo/some:/mnt/foo/some:rw' + BUCKET | PATH | PARENT | EXPECTED | MOUNTS + 'foo' | '/' | false | Paths.get('/mnt/disks/foo') | '/mnt/disks/foo:/mnt/disks/foo:rw' + 'foo' | '/some/dir' | false | Paths.get('/mnt/disks/foo/some/dir') | '/mnt/disks/foo/some/dir:/mnt/disks/foo/some/dir:rw' + 'foo' | '/some/dir' | true | Paths.get('/mnt/disks/foo/some/dir') | '/mnt/disks/foo/some:/mnt/disks/foo/some:rw' } def 'should compute volume mounts' () { @@ -54,26 +54,26 @@ class GoogleBatchScriptLauncherTest extends Specification{ def PATH3 = CloudStorageFileSystem.forBucket('omega').getPath('/data/sample3.bam') expect: - launcher.toContainerMount(PATH1) == Paths.get('/mnt/alpha/data/sample1.bam') - launcher.toContainerMount(PATH2) == Paths.get('/mnt/alpha/data/sample2.bam') - launcher.toContainerMount(PATH3) == Paths.get('/mnt/omega/data/sample3.bam') + launcher.toContainerMount(PATH1) == Paths.get('/mnt/disks/alpha/data/sample1.bam') + launcher.toContainerMount(PATH2) == Paths.get('/mnt/disks/alpha/data/sample2.bam') + launcher.toContainerMount(PATH3) == Paths.get('/mnt/disks/omega/data/sample3.bam') and: def containerMounts = launcher.getContainerMounts() and: containerMounts.size() == 2 - containerMounts[0] == '/mnt/alpha/data:/mnt/alpha/data:rw' - containerMounts[1] == '/mnt/omega/data/sample3.bam:/mnt/omega/data/sample3.bam:rw' + containerMounts[0] == '/mnt/disks/alpha/data:/mnt/disks/alpha/data:rw' + containerMounts[1] == '/mnt/disks/omega/data/sample3.bam:/mnt/disks/omega/data/sample3.bam:rw' and: def volumes = launcher.getVolumes() and: volumes.size() == 2 volumes[0].getGcs().getRemotePath() == 'alpha' - volumes[0].getMountPath() == '/mnt/alpha' + volumes[0].getMountPath() == '/mnt/disks/alpha' volumes[0].getMountOptionsList() == ['-o rw,allow_other', '-implicit-dirs'] volumes[1].getGcs().getRemotePath() == 'omega' - volumes[1].getMountPath() == '/mnt/omega' + volumes[1].getMountPath() == '/mnt/disks/omega' volumes[1].getMountOptionsList() == ['-o rw,allow_other', '-implicit-dirs'] } diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy index 0b8b2bd468..e2e4cf97f9 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy @@ -71,10 +71,10 @@ class GoogleBatchTaskHandlerTest extends Specification { taskGroup.getTaskSpec().getComputeResource().getMemoryMib() == 0 taskGroup.getTaskSpec().getMaxRunDuration().getSeconds() == 0 and: - runnable.getContainer().getCommandsList().join(' ') == '/bin/bash -o pipefail -c trap "{ cp .command.log /mnt/foo/scratch/.command.log; }" ERR; /bin/bash /mnt/foo/scratch/.command.run 2>&1 | tee .command.log' + runnable.getContainer().getCommandsList().join(' ') == '/bin/bash -o pipefail -c trap "{ cp .command.log /mnt/disks/foo/scratch/.command.log; }" ERR; /bin/bash /mnt/disks/foo/scratch/.command.run 2>&1 | tee .command.log' runnable.getContainer().getImageUri() == CONTAINER_IMAGE runnable.getContainer().getOptions() == '' - runnable.getContainer().getVolumesList() == ['/mnt/foo/scratch:/mnt/foo/scratch:rw'] + runnable.getContainer().getVolumesList() == ['/mnt/disks/foo/scratch:/mnt/disks/foo/scratch:rw'] and: instancePolicy.getAcceleratorsCount() == 0 instancePolicy.getMachineType() == '' @@ -146,10 +146,10 @@ class GoogleBatchTaskHandlerTest extends Specification { taskGroup.getTaskSpec().getComputeResource().getMemoryMib() == MEM.toMega() taskGroup.getTaskSpec().getMaxRunDuration().getSeconds() == TIMEOUT.seconds and: - runnable.getContainer().getCommandsList().join(' ') == '/bin/bash -o pipefail -c trap "{ cp .command.log /mnt/foo/scratch/.command.log; }" ERR; /bin/bash /mnt/foo/scratch/.command.run 2>&1 | tee .command.log' + runnable.getContainer().getCommandsList().join(' ') == '/bin/bash -o pipefail -c trap "{ cp .command.log /mnt/disks/foo/scratch/.command.log; }" ERR; /bin/bash /mnt/disks/foo/scratch/.command.run 2>&1 | tee .command.log' runnable.getContainer().getImageUri() == CONTAINER_IMAGE runnable.getContainer().getOptions() == CONTAINER_OPTS - runnable.getContainer().getVolumesList() == ['/mnt/foo/scratch:/mnt/foo/scratch:rw'] + runnable.getContainer().getVolumesList() == ['/mnt/disks/foo/scratch:/mnt/disks/foo/scratch:rw'] and: instancePolicy.getAccelerators(0).getCount() == 1 instancePolicy.getAccelerators(0).getType() == ACCELERATOR.type From d64eeffcb0b4c7d2d2773f0492fff15e05399e21 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 11:17:52 +0200 Subject: [PATCH 06/12] Add tags propagation to aws batch Signed-off-by: Paolo Di Tommaso --- .../cloud/aws/batch/AwsBatchTaskHandler.groovy | 10 +++++----- .../cloud/aws/batch/AwsBatchTaskHandlerTest.groovy | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy index 14e712e8ab..da61ecc39f 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchTaskHandler.groovy @@ -368,10 +368,6 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler getResourceLabels(TaskRun task){ - task.config.getResourceLabels() ?: Collections.emptyMap() - } - /** * Maps a docker container image to a Batch job definition name * @@ -652,11 +648,15 @@ class AwsBatchTaskHandler extends TaskHandler implements BatchHandler Date: Mon, 26 Sep 2022 16:22:05 +0200 Subject: [PATCH 07/12] Fix task resume when updating fusion layer [ci fast] Signed-off-by: Paolo Di Tommaso --- .../seqera/wave/plugin/ContainerConfig.groovy | 17 +++++++ .../seqera/wave/plugin/ContainerLayer.groovy | 31 ++++++++++++ .../io/seqera/wave/plugin/WaveAssets.groovy | 2 +- .../wave/plugin/config/WaveConfig.groovy | 2 +- .../wave/plugin/ContainerConfigTest.groovy | 50 +++++++++++++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerConfig.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerConfig.groovy index c22cd5e2dd..c1eed4ae88 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerConfig.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerConfig.groovy @@ -20,6 +20,8 @@ package io.seqera.wave.plugin import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString +import nextflow.util.CacheHelper + /** * Model a container configuration * @@ -91,4 +93,19 @@ class ContainerConfig { if( right ) result.addAll(right) return result } + + String fingerprint() { + final allMeta = new ArrayList() + allMeta.add( entrypoint ?: 'no-entry' ) + allMeta.add( cmd ?: 'no-cmd' ) + allMeta.add( env ?: 'no-env' ) + allMeta.add( workingDir ?: 'no-workdir') + final layers0 = layers ?: Collections.emptyList() + + for( ContainerLayer it : layers0 ) { + if( !it.skipHashing ) + allMeta.add(it.fingerprint()) + } + return CacheHelper.hasher(allMeta).hash().toString() + } } diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerLayer.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerLayer.groovy index b82403fd4b..ddaa20797c 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerLayer.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/ContainerLayer.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.plugin import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.ToString +import nextflow.util.CacheHelper /** * Model a container layer meta-info @@ -30,15 +31,45 @@ import groovy.transform.ToString @ToString(includeNames = true, includePackage = false) @CompileStatic class ContainerLayer { + /** + * the layer location, it can be either `http:` or `https:` prefixed URI + * or a `data:` pseudo-protocol followed by a base64 encoded tar gzipped layer payload + */ String location + + /** + * The layer gzip sha256 checksum + */ String gzipDigest + + /** + * The layer gzip size in bytes + */ Integer gzipSize + + /** + * The layer tar sha256 checksum + */ String tarDigest + /** + * When {@code this layer is not added in the final config fingerprint} + */ + Boolean skipHashing + void validate() { if( !location ) throw new IllegalArgumentException("Missing layer location") if( !gzipDigest ) throw new IllegalArgumentException("Missing layer gzip digest") if( !gzipSize ) throw new IllegalArgumentException("Missing layer gzip size") if( !tarDigest ) throw new IllegalArgumentException("Missing layer tar digest") } + + String fingerprint() { + final allMeta = new ArrayList() + allMeta.add( location ?: 'no-location' ) + allMeta.add( gzipDigest ?: 'no-gzipDigest' ) + allMeta.add( gzipSize ?: 0 ) + allMeta.add( tarDigest ?: 'no-tarDigest') + return CacheHelper.hasher(allMeta).hash().toString() + } } diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy index 1a94708ee2..e99187ed65 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveAssets.groovy @@ -64,7 +64,7 @@ class WaveAssets { final allMeta = new ArrayList(10) allMeta.add( this.containerImage ) allMeta.add( this.moduleResources?.fingerprint() ) - allMeta.add( this.containerConfig?.hashCode() ) + allMeta.add( this.containerConfig?.fingerprint() ) allMeta.add( this.dockerFileContent ) allMeta.add( this.condaFile ) allMeta.add( this.projectResources?.fingerprint() ) diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy index 10b4e75cb8..3b4a4e52e6 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy @@ -111,7 +111,7 @@ class WaveConfig { } List containerConfigUrl() { - return containerConfigUrl + return containerConfigUrl ?: Collections.emptyList() } Duration tokensCacheMaxDuration() { diff --git a/plugins/nf-wave/src/test/io/seqera/wave/plugin/ContainerConfigTest.groovy b/plugins/nf-wave/src/test/io/seqera/wave/plugin/ContainerConfigTest.groovy index 5b65711109..cead6646a0 100644 --- a/plugins/nf-wave/src/test/io/seqera/wave/plugin/ContainerConfigTest.groovy +++ b/plugins/nf-wave/src/test/io/seqera/wave/plugin/ContainerConfigTest.groovy @@ -111,4 +111,54 @@ class ContainerConfigTest extends Specification { and: [new ContainerLayer(location: 'http://x')] | [new ContainerLayer(location: 'http://y'),new ContainerLayer(location: 'http://y')] | [new ContainerLayer(location: 'http://x'), new ContainerLayer(location: 'http://y'),new ContainerLayer(location: 'http://y')] } + + def 'should compute config fingerprint' () { + given: + def config1 = new ContainerConfig(entrypoint: ['entry.sh'], cmd: ['the.cmd'], env: ['x=2'], workingDir: '/foo') + def config2 = new ContainerConfig(entrypoint: ['entry.sh'], cmd: ['the.cmd'], env: ['x=2'], workingDir: '/foo') + def config3 = new ContainerConfig(entrypoint: ['entry.sh'], cmd: ['the.cmd'], env: ['x=2'], workingDir: '/bar') + and: + def layer1 = new ContainerLayer('http://this/that', 'abd', 100, 'efg') + def layer2 = new ContainerLayer('http://this/that', 'abd', 100, 'efg') + def layer3 = new ContainerLayer('http://xxx/yyy', 'abd', 200, 'efg') + and: + def fusion1 = new ContainerLayer('abc', 'xyz', 300, 'efg', true) + def fusion2 = new ContainerLayer('efg', 'pqr', 400, 'efg', true) + + expect: + config1.fingerprint() == config2.fingerprint() + config1.fingerprint() != config3.fingerprint() + + when: + config1.appendLayer(layer1) + config2.appendLayer(layer2) + then: + // layers have the same fingerprint + layer1.fingerprint() == layer2.fingerprint() + // configs have the same fingerprint + config1.fingerprint() == config2.fingerprint() + + when: + config1.layers.clear() + config2.layers.clear() + and: + config1.appendLayer(layer1) + config2.appendLayer(layer3) + then: + // different layer fingerprint cause the config to have different fingerprints + layer1.fingerprint() != layer3.fingerprint() + config1.fingerprint() != config2.fingerprint() + + when: + config1.layers.clear() + config2.layers.clear() + and: + config1.appendLayer(fusion1) + config2.appendLayer(fusion2) + then: + // the layer have different fingerprint BUT the `skipHashing` + // makes the config to ignore it + fusion1.fingerprint() != fusion2.fingerprint() + config1.fingerprint() == config2.fingerprint() + } } From f9ff14d1440ff6988a90c5b5d1c2a6fe2ab46e5d Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 17:09:18 +0200 Subject: [PATCH 08/12] Bump nf-amazon@1.10.6 Signed-off-by: Paolo Di Tommaso --- plugins/nf-amazon/changelog.txt | 3 +++ plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/nf-amazon/changelog.txt b/plugins/nf-amazon/changelog.txt index 70b7631c40..6b3c258227 100644 --- a/plugins/nf-amazon/changelog.txt +++ b/plugins/nf-amazon/changelog.txt @@ -1,5 +1,8 @@ nf-amazon changelog =================== +1.10.6 - 26 Sep 2022 +- Add tags propagation to AWS Batch [d64eeffc] + 1.10.5 - 20 Sep 2022 - Fix AWS S3 copy object [b3b90d23] diff --git a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF index d774209727..370b71988c 100644 --- a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.aws.AmazonPlugin Plugin-Id: nf-amazon -Plugin-Version: 1.10.5 +Plugin-Version: 1.10.6 Plugin-Provider: Seqera Labs Plugin-Requires: >=22.09.2-edge From ff04460edfd5a31d5ab2a41e4e88c31aefb7022f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 17:09:36 +0200 Subject: [PATCH 09/12] Bump nf-wave@0.4.4 Signed-off-by: Paolo Di Tommaso --- plugins/nf-wave/src/resources/META-INF/MANIFEST.MF | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF index 1897e3be7d..ef00816fe7 100644 --- a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: io.seqera.wave.plugin.WavePlugin Plugin-Id: nf-wave -Plugin-Version: 0.4.3 +Plugin-Version: 0.4.4 Plugin-Provider: Seqera Labs Plugin-Requires: >=22.09.0-edge From 2df94b01622d2a8258a92127d5b41fadc17fa4ea Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 17:10:43 +0200 Subject: [PATCH 10/12] Bump nf-google@1.4.4 Signed-off-by: Paolo Di Tommaso --- plugins/nf-google/changelog.txt | 4 ++++ plugins/nf-google/src/resources/META-INF/MANIFEST.MF | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/nf-google/changelog.txt b/plugins/nf-google/changelog.txt index 73d5e9eec3..c3587e93b4 100644 --- a/plugins/nf-google/changelog.txt +++ b/plugins/nf-google/changelog.txt @@ -1,5 +1,9 @@ nf-google changelog =================== +1.4.3 - 26 Sep 2022 +- Update Google Batch mount point with the requirements [5aec28ac] +- Apply GCP resourceLabels to the VirtualMachine (#3234) [2275c03c] + 1.4.2 - 1 Sep 2022 - Add support for resource labels for google batch (#3168) (#3170) [2d24917b] (#2853) [5d0b7c35] - Add support for project resources [c2ad6566] diff --git a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF index 847f4d2ffd..89565e592a 100644 --- a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.google.GoogleCloudPlugin Plugin-Id: nf-google -Plugin-Version: 1.4.3 +Plugin-Version: 1.4.4 Plugin-Provider: Seqera Labs Plugin-Requires: >=22.09.0-edge From 967d7c14d9d350c49164f8b4e202c75fb4db5ea2 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 17:16:37 +0200 Subject: [PATCH 11/12] Update changelog Signed-off-by: Paolo Di Tommaso --- changelog.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/changelog.txt b/changelog.txt index 2a32c435ab..663537de73 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,16 @@ NEXTFLOW CHANGE-LOG =================== +22.09.6-edge - 26 Sep 2022 +- Add SocketTimeoutException to k8s client request retry [527e0d5d] +- Add MaxErrorRetry to K8s config [58be2128] +- Add tags propagation to AWS Batch [d64eeffc] +- Fix task resume when updating fusion layer [f38fd2db] +- Fix Channel merge still deprecated for DSL2 (#3220) [d27384d2] +- Apply GCP resourceLabels to the VirtualMachine (#3234) [2275c03c] +- Update Google Batch mount point with the requirements [5aec28ac] +- Improve wave error reporting [73842215] +- Bump fusion 0.4.x [26f1f896] + 22.09.5-edge - 21 Sep 2022 - Use default wave strategy [abbfa7f4] - Handle errors reported by tower report writer [0e814647] From 021d898944428a8883fd73f3058aafc2043b0cca Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 26 Sep 2022 17:17:01 +0200 Subject: [PATCH 12/12] [release 22.09.6-edge] Update timestamp and build number [ci fast] Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- docs/conf.py | 2 +- .../nextflow/src/main/resources/META-INF/plugins-info.txt | 6 +++--- modules/nf-commons/src/main/nextflow/Const.groovy | 6 +++--- nextflow | 2 +- nextflow.md5 | 2 +- nextflow.sha1 | 2 +- nextflow.sha256 | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/VERSION b/VERSION index f6a5ccbd87..ed39177393 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -22.09.5-edge +22.09.6-edge diff --git a/docs/conf.py b/docs/conf.py index 39810eff46..e2c4bce27a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,7 +54,7 @@ # The short X.Y version. version = '22.09' # The full version, including alpha/beta/rc tags. -release = '22.09.5-edge' +release = '22.09.6-edge' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt index 97ddcf1f3b..90f4ba5840 100644 --- a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt +++ b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt @@ -1,7 +1,7 @@ -nf-amazon@1.10.5 -nf-wave@0.4.3 +nf-amazon@1.10.6 +nf-wave@0.4.4 nf-console@1.0.4 -nf-google@1.4.3 +nf-google@1.4.4 nf-azure@0.14.2 nf-tower@1.5.4 nf-ga4gh@1.0.4 diff --git a/modules/nf-commons/src/main/nextflow/Const.groovy b/modules/nf-commons/src/main/nextflow/Const.groovy index 40f27213b5..96079233e1 100644 --- a/modules/nf-commons/src/main/nextflow/Const.groovy +++ b/modules/nf-commons/src/main/nextflow/Const.groovy @@ -53,17 +53,17 @@ class Const { /** * The application version */ - static public final String APP_VER = "22.09.5-edge" + static public final String APP_VER = "22.09.6-edge" /** * The app build time as linux/unix timestamp */ - static public final long APP_TIMESTAMP = 1663747945997 + static public final long APP_TIMESTAMP = 1664205183116 /** * The app build number */ - static public final int APP_BUILDNUM = 5799 + static public final int APP_BUILDNUM = 5802 /** * The app build time string relative to UTC timezone diff --git a/nextflow b/nextflow index fab50bc10e..c7c64f3b90 100755 --- a/nextflow +++ b/nextflow @@ -16,7 +16,7 @@ # limitations under the License. [[ "$NXF_DEBUG" == 'x' ]] && set -x -NXF_VER=${NXF_VER:-'22.09.5-edge'} +NXF_VER=${NXF_VER:-'22.09.6-edge'} NXF_ORG=${NXF_ORG:-'nextflow-io'} NXF_HOME=${NXF_HOME:-$HOME/.nextflow} NXF_PROT=${NXF_PROT:-'https'} diff --git a/nextflow.md5 b/nextflow.md5 index fd01ebfebf..0f0f0d2496 100644 --- a/nextflow.md5 +++ b/nextflow.md5 @@ -1 +1 @@ -3ba1581357bbc9874fceb1f7a995bc27 +70fd600b06967e1b03c76ede2545eea9 diff --git a/nextflow.sha1 b/nextflow.sha1 index 360f4375a7..a43bf355f1 100644 --- a/nextflow.sha1 +++ b/nextflow.sha1 @@ -1 +1 @@ -35aff3b925492727d8ce1b4f410fb30f3aaada17 +1decc861772398b1bb2414ef0d594c6d266d856e diff --git a/nextflow.sha256 b/nextflow.sha256 index c1641652f6..4b7452f5b2 100644 --- a/nextflow.sha256 +++ b/nextflow.sha256 @@ -1 +1 @@ -511fe4e911c96d6d379739f86bde066d02b18acfaa55edb3f7306d78a31fa1ba +ce1612e5ee1b7713c30c63d98dac44761b915587ff25f82d0c6228cdece11183