[GH-93] Failed to mount s3 in us-east-1

The endpoint for AWS us-east-1 is different with other regions.
MMC is not able to find the correct endpoint by itself. As a work
around, we need the user to supply the endpoint in the configuration
`aws.client.endpoint`.  The value should be `https://s3.us-east-1.amazonaws.com`

README.md

@@ -201,6 +201,17 @@ aws {
 }
 ```
 
+Due to a compatibility issue between MMC and AWS, when using the us-east-1 region, 
+you must explicitly specify the endpoint as `https://s3.us-east-1.amazonaws.com`. 
+For example:
+```groovy
+aws {
+    client {
+        endpoint = 'https://s3.us-east-1.amazonaws.com'
+    }
+}
+```
+
 If you are sure that the workflow file is properly composed, it's recommended to
 set proper error strategy and retry limit in the process scope to make sure
 the workflow can be completed.

plugins/nf-float/src/main/com/memverge/nextflow/Global.groovy

@@ -48,13 +48,26 @@ class AWSCred {
     String accessKey
     String secretKey
     String token
+    String endpoint
 
     AWSCred(String accessKey, String secretKey, String token = null) {
         this.accessKey = accessKey
         this.secretKey = secretKey
         this.token = token
     }
 
+    def SetEndpoint(Object object) {
+        if (!object instanceof String) {
+            return
+        }
+        String endpoint = object as String
+        endpoint = endpoint.trim()
+        if (!endpoint.startsWith("https://")) {
+            endpoint = "https://" + endpoint
+        }
+        this.endpoint = endpoint
+    }
+
     Boolean isValid() {
         return accessKey && secretKey
     }
@@ -140,6 +153,9 @@ class AWSCred {
         if (token) {
             ret.add("token=${token}")
         }
+        if (endpoint) {
+            ret.add("endpoint=${endpoint}")
+        }
         return ret
     }
 
@@ -296,16 +312,22 @@ class Global {
         AWSCred ret
 
         if (config && config.aws instanceof Map) {
-            key = ((Map) config.aws).accessKey
-            secret = ((Map) config.aws).secretKey
-            token = ((Map) config.aws).token
+            final awsConfig = config.aws as Map
+            key = awsConfig.accessKey
+            secret = awsConfig.secretKey
+            token = awsConfig.token
 
             ret = new AWSCred(key, secret, token)
+            if (awsConfig.client instanceof Map) {
+                final awsCliConfig = awsConfig.client as Map
+                if (awsCliConfig.endpoint) {
+                    ret.SetEndpoint(awsCliConfig.endpoint)
+                }
+            }
             if (ret.isValid()) {
                 log.debug "[FLOAT] Using AWS credentials defined in nextflow config file"
                 return ret
             }
-
         }
 
         // as define by amazon doc

plugins/nf-float/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: com.memverge.nextflow.FloatPlugin
 Plugin-Id: nf-float
-Plugin-Version: 0.4.4
+Plugin-Version: 0.4.5
 Plugin-Provider: MemVerge Corp.
 Plugin-Requires: >=23.04.0

plugins/nf-float/src/test/com/memverge/nextflow/FloatConfTest.groovy

@@ -132,6 +132,24 @@ class FloatConfTest extends BaseTest {
                 's3://bucket:/bucket'
     }
 
+    def "get s3 data volume with endpoint"() {
+        given:
+        def conf = [
+                aws: [
+                        accessKey: 'ak0',
+                        secretKey: 'sk0',
+                        client   : [
+                                endpoint: 's3.ep']]]
+        when:
+        def fConf = FloatConf.getConf(conf)
+        def workDir = new URI('s3://bucket/work/dir')
+        def volume = fConf.getWorkDirVol(workDir)
+
+        then:
+        volume == '[mode=rw,accesskey=ak0,secret=sk0,endpoint=https://s3.ep]' +
+                's3://bucket:/bucket'
+    }
+
     def "get s3 credentials from env 0"() {
         given:
         setEnv('AWS_ACCESS_KEY_ID', 'aak_id')
@@ -288,7 +306,7 @@ class FloatConfTest extends BaseTest {
 
 
 class DataVolumeTest extends BaseTest {
-    def "parse nfs volume" (){
+    def "parse nfs volume"() {
         given:
         def nfs = "nfs://1.2.3.4/my/dir:/mnt/point"
 
@@ -300,7 +318,7 @@ class DataVolumeTest extends BaseTest {
         vol.toString() == nfs
     }
 
-    def "parse s3 without credentials" () {
+    def "parse s3 without credentials"() {
         given:
         def s3 = "[mode=rw]s3://1.2.3.4/my/dir:/mnt/point"
 
@@ -312,7 +330,7 @@ class DataVolumeTest extends BaseTest {
         vol.toString() == s3
     }
 
-    def "existing s3 credentials" () {
+    def "existing s3 credentials"() {
         given:
         def s3 = "[accessKey=a,mode=rw,secret=s]s3://1.2.3.4/my/dir:/mnt/point"
 
@@ -325,7 +343,7 @@ class DataVolumeTest extends BaseTest {
         vol.toString() == s3
     }
 
-    def "update s3 credentials" () {
+    def "update s3 credentials"() {
         given:
         def s3 = "[secret=s]s3://1.2.3.4/my/dir:/mnt/point"
 
@@ -338,7 +356,7 @@ class DataVolumeTest extends BaseTest {
         vol.toString() == "[accesskey=x,mode=rw,secret=y]s3://1.2.3.4/my/dir:/mnt/point"
     }
 
-    def "update s3 credentials" () {
+    def "update s3 credentials"() {
         given:
         def s3 = "[mode=rw,secret=s]s3://1.2.3.4/my/dir:/mnt/point"