diff --git a/README.md b/README.md index 43d6021..0ca7cef 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,17 @@ aws { } ``` +Due to a compatibility issue between MMC and AWS, when using the us-east-1 region, +you must explicitly specify the endpoint as `https://s3.us-east-1.amazonaws.com`. +For example: +```groovy +aws { + client { + endpoint = 'https://s3.us-east-1.amazonaws.com' + } +} +``` + If you are sure that the workflow file is properly composed, it's recommended to set proper error strategy and retry limit in the process scope to make sure the workflow can be completed. diff --git a/plugins/nf-float/src/main/com/memverge/nextflow/Global.groovy b/plugins/nf-float/src/main/com/memverge/nextflow/Global.groovy index 77571a2..6e16c34 100644 --- a/plugins/nf-float/src/main/com/memverge/nextflow/Global.groovy +++ b/plugins/nf-float/src/main/com/memverge/nextflow/Global.groovy @@ -48,6 +48,7 @@ class AWSCred { String accessKey String secretKey String token + String endpoint AWSCred(String accessKey, String secretKey, String token = null) { this.accessKey = accessKey @@ -55,6 +56,18 @@ class AWSCred { this.token = token } + def SetEndpoint(Object object) { + if (!object instanceof String) { + return + } + String endpoint = object as String + endpoint = endpoint.trim() + if (!endpoint.startsWith("https://")) { + endpoint = "https://" + endpoint + } + this.endpoint = endpoint + } + Boolean isValid() { return accessKey && secretKey } @@ -140,6 +153,9 @@ class AWSCred { if (token) { ret.add("token=${token}") } + if (endpoint) { + ret.add("endpoint=${endpoint}") + } return ret } @@ -296,16 +312,22 @@ class Global { AWSCred ret if (config && config.aws instanceof Map) { - key = ((Map) config.aws).accessKey - secret = ((Map) config.aws).secretKey - token = ((Map) config.aws).token + final awsConfig = config.aws as Map + key = awsConfig.accessKey + secret = awsConfig.secretKey + token = awsConfig.token ret = new AWSCred(key, secret, token) + if (awsConfig.client instanceof Map) { + final awsCliConfig = awsConfig.client as Map + if (awsCliConfig.endpoint) { + ret.SetEndpoint(awsCliConfig.endpoint) + } + } if (ret.isValid()) { log.debug "[FLOAT] Using AWS credentials defined in nextflow config file" return ret } - } // as define by amazon doc diff --git a/plugins/nf-float/src/resources/META-INF/MANIFEST.MF b/plugins/nf-float/src/resources/META-INF/MANIFEST.MF index e994d09..2f858cf 100644 --- a/plugins/nf-float/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-float/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: com.memverge.nextflow.FloatPlugin Plugin-Id: nf-float -Plugin-Version: 0.4.4 +Plugin-Version: 0.4.5 Plugin-Provider: MemVerge Corp. Plugin-Requires: >=23.04.0 diff --git a/plugins/nf-float/src/test/com/memverge/nextflow/FloatConfTest.groovy b/plugins/nf-float/src/test/com/memverge/nextflow/FloatConfTest.groovy index 4937c81..1d5a4b8 100644 --- a/plugins/nf-float/src/test/com/memverge/nextflow/FloatConfTest.groovy +++ b/plugins/nf-float/src/test/com/memverge/nextflow/FloatConfTest.groovy @@ -132,6 +132,24 @@ class FloatConfTest extends BaseTest { 's3://bucket:/bucket' } + def "get s3 data volume with endpoint"() { + given: + def conf = [ + aws: [ + accessKey: 'ak0', + secretKey: 'sk0', + client : [ + endpoint: 's3.ep']]] + when: + def fConf = FloatConf.getConf(conf) + def workDir = new URI('s3://bucket/work/dir') + def volume = fConf.getWorkDirVol(workDir) + + then: + volume == '[mode=rw,accesskey=ak0,secret=sk0,endpoint=https://s3.ep]' + + 's3://bucket:/bucket' + } + def "get s3 credentials from env 0"() { given: setEnv('AWS_ACCESS_KEY_ID', 'aak_id') @@ -288,7 +306,7 @@ class FloatConfTest extends BaseTest { class DataVolumeTest extends BaseTest { - def "parse nfs volume" (){ + def "parse nfs volume"() { given: def nfs = "nfs://1.2.3.4/my/dir:/mnt/point" @@ -300,7 +318,7 @@ class DataVolumeTest extends BaseTest { vol.toString() == nfs } - def "parse s3 without credentials" () { + def "parse s3 without credentials"() { given: def s3 = "[mode=rw]s3://1.2.3.4/my/dir:/mnt/point" @@ -312,7 +330,7 @@ class DataVolumeTest extends BaseTest { vol.toString() == s3 } - def "existing s3 credentials" () { + def "existing s3 credentials"() { given: def s3 = "[accessKey=a,mode=rw,secret=s]s3://1.2.3.4/my/dir:/mnt/point" @@ -325,7 +343,7 @@ class DataVolumeTest extends BaseTest { vol.toString() == s3 } - def "update s3 credentials" () { + def "update s3 credentials"() { given: def s3 = "[secret=s]s3://1.2.3.4/my/dir:/mnt/point" @@ -338,7 +356,7 @@ class DataVolumeTest extends BaseTest { vol.toString() == "[accesskey=x,mode=rw,secret=y]s3://1.2.3.4/my/dir:/mnt/point" } - def "update s3 credentials" () { + def "update s3 credentials"() { given: def s3 = "[mode=rw,secret=s]s3://1.2.3.4/my/dir:/mnt/point"