Fix s3fs region access, s3 url styles

CHANGELOG.unreleased.md

@@ -23,6 +23,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Fixed a bug where it was possible to create invalid an state by deleting teams that are referenced elsewhere. [6664](https://github.com/scalableminds/webknossos/pull/6664)
 - Miscellaneous fixes for the new folder UI. [#6674](https://github.com/scalableminds/webknossos/pull/6674)
 - Fixed import of remote datasets with multiple layers and differing resolution pyramid. #[6670](https://github.com/scalableminds/webknossos/pull/6670)
+- Fixed access of remote datasets using the Amazon S3 protocol [#6679](https://github.com/scalableminds/webknossos/pull/6679)
 
 ### Removed
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/AmazonS3Factory.java

@@ -53,7 +53,8 @@ public AmazonS3 getAmazonS3Client(URI uri, Properties props) {
             .standard()
             .withCredentials(getCredentialsProvider(props))
             .withClientConfiguration(getClientConfiguration(props))
-            .withEndpointConfiguration(getEndpointConfiguration(uri))
+            .withRegion(Regions.DEFAULT_REGION)
+            .withForceGlobalBucketAccessEnabled(true)
             .build();
     }
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystem.java

@@ -24,15 +24,15 @@ public class S3FileSystem extends FileSystem implements Comparable<S3FileSystem>
     private final S3FileSystemProvider provider;
     private final String key;
     private final AmazonS3 client;
-    private final String endpoint;
     private int cache;
+    private final String bucket;
 
-    public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint) {
+    public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String bucket) {
         this.provider = provider;
         this.key = key;
         this.client = client;
-        this.endpoint = endpoint;
         this.cache = 60000; // 1 minute cache for the s3Path
+        this.bucket = bucket;
     }
 
     @Override
@@ -44,6 +44,14 @@ public String getKey() {
         return key;
     }
 
+    public String getBucket() {
+        return bucket;
+    }
+
+    public Boolean hasBucket() {
+        return bucket != null;
+    }
+
     @Override
     public void close() throws IOException {
         this.provider.close(this);
@@ -89,6 +97,10 @@ public Set<String> supportedFileAttributeViews() {
 
     @Override
     public S3Path getPath(String first, String... more) {
+
+        if (hasBucket() && first.startsWith("/")) {
+          first = "/" + this.bucket + first;
+        }
         if (more.length == 0) {
             return new S3Path(this, first);
         }
@@ -115,16 +127,6 @@ public AmazonS3 getClient() {
         return client;
     }
 
-    /**
-     * get the endpoint associated with this fileSystem.
-     *
-     * @return string
-     * @see <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html">http://docs.aws.amazon.com/general/latest/gr/rande.html</a>
-     */
-    public String getEndpoint() {
-        return endpoint;
-    }
-
     public String[] key2Parts(String keyParts) {
         String[] parts = keyParts.split(S3Path.PATH_SEPARATOR);
         String[] split = new String[parts.length];
@@ -139,7 +141,7 @@ public String[] key2Parts(String keyParts) {
     public int hashCode() {
         final int prime = 31;
         int result = 1;
-        result = prime * result + ((endpoint == null) ? 0 : endpoint.hashCode());
+        result = prime * result;
         result = prime * result + ((key == null) ? 0 : key.hashCode());
         return result;
     }
@@ -153,11 +155,6 @@ public boolean equals(Object obj) {
         if (!(obj instanceof S3FileSystem))
             return false;
         S3FileSystem other = (S3FileSystem) obj;
-        if (endpoint == null) {
-            if (other.endpoint != null)
-                return false;
-        } else if (!endpoint.equals(other.endpoint))
-            return false;
         if (key == null) {
             if (other.key != null)
                 return false;

webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystemProvider.java

@@ -549,21 +549,39 @@ public void setAttribute(Path path, String attribute, Object value, LinkOption..
      */
     public S3FileSystem createFileSystem(URI uri, Properties props) {
         URI uriWithNormalizedHost = resolveShortcutHost(uri);
-        return new S3FileSystem(this, getFileSystemKey(uriWithNormalizedHost, props), getAmazonS3(uriWithNormalizedHost, props), uriWithNormalizedHost.getHost());
+        String bucket = hostBucketFromUri(uri);
+        return new S3FileSystem(this, getFileSystemKey(uriWithNormalizedHost, props), getAmazonS3(uriWithNormalizedHost, props), bucket);
     }
 
     private URI resolveShortcutHost(URI uri) {
       String host = uri.getHost();
-      if (!uri.getHost().contains(".")) {
-        host += ".s3.amazonaws.com";
+      String newHost = host;
+      String bucketPrefix = "";
+      if (!host.contains(".")) { // assume host is omitted from uri, shortcut form s3://bucket/key
+        newHost = "s3.amazonaws.com";
+        bucketPrefix = "/" + host;
+      } else if (host.endsWith(".s3.amazonaws.com")) {
+        newHost = "s3.amazonaws.com";
+        bucketPrefix = "/" + host.substring(0, host.length() - ".s3.amazonaws.com".length());
       }
       try {
-        return new URI(uri.getScheme(), uri.getUserInfo(), host, uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment());
+        return new URI(uri.getScheme(), uri.getUserInfo(), newHost, uri.getPort(), bucketPrefix + uri.getPath(), uri.getQuery(), uri.getFragment());
       } catch (URISyntaxException e) {
         return uri;
       }
     }
 
+
+    private String hostBucketFromUri(URI uri) {
+      String host = uri.getHost();
+      if (!host.contains(".")) { // assume host is omitted from uri, shortcut form s3://bucket/key
+        return host;
+      } else if (host.endsWith(".s3.amazonaws.com")) {
+        return host.substring(0, host.length() - ".s3.amazonaws.com".length());
+      }
+      return null;
+    }
+
     protected AmazonS3 getAmazonS3(URI uri, Properties props) {
         return new AmazonS3Factory().getAmazonS3Client(uri, props);
     }