Update to the notice and refactor the createAzureFs

apache · Aug 4, 2021 · 3ab85bf · 3ab85bf
1 parent 5c57c5c
commit 3ab85bf
Show file tree

Hide file tree

Showing 9 changed files with 92 additions and 36 deletions.
diff --git a/...s/flink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/ABFSAzureFSFactory.java b/...s/flink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/ABFSAzureFSFactory.java
@@ -18,11 +18,25 @@
 
 package org.apache.flink.fs.azurefs;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+
 /** Abfs azureFs implementation. */
 public class ABFSAzureFSFactory extends AbstractAzureFSFactory {
 
     @Override
     public String getScheme() {
         return "abfs";
     }
+
+    @Override
+    FileSystem createAzureFS(URI fsUri, Configuration hadoopConfig) throws IOException {
+        org.apache.hadoop.fs.FileSystem azureFS = new AzureBlobFileSystem();
+        azureFS.initialize(fsUri, hadoopConfig);
+        return azureFS;
+    }
 }
diff --git a/...ink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/AbstractAzureFSFactory.java b/...ink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/AbstractAzureFSFactory.java
@@ -24,8 +24,6 @@
 import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem;
 import org.apache.flink.runtime.util.HadoopConfigLoader;
 
-import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
-import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -73,30 +71,14 @@ public void configure(Configuration config) {
         configLoader.setFlinkConfig(config);
     }
 
+    abstract org.apache.hadoop.fs.FileSystem createAzureFS(
+            URI fsUri, org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException;
+
     @Override
     public FileSystem create(URI fsUri) throws IOException {
         checkNotNull(fsUri, "passed file system URI object should not be null");
         LOG.info("Trying to load and instantiate Azure File System for {}", fsUri);
-        return new HadoopFileSystem(createInitializedAzureFS(fsUri, flinkConfig));
-    }
-
-    // uri is of the form: wasb(s)://[email protected]/testDir (or)
-    // abfs(s):////[email protected]/testDir
-    private org.apache.hadoop.fs.FileSystem createInitializedAzureFS(
-            URI fsUri, Configuration flinkConfig) throws IOException {
         org.apache.hadoop.conf.Configuration hadoopConfig = configLoader.getOrLoadHadoopConfig();
-        String scheme = fsUri.getScheme();
-
-        if (scheme.startsWith("wasb")) {
-            LOG.info("Trying to initialize hadoop filesystem for {}.", scheme);
-            org.apache.hadoop.fs.FileSystem azureFS = new NativeAzureFileSystem();
-            azureFS.initialize(fsUri, hadoopConfig);
-            return azureFS;
-        } else {
-            LOG.info("Trying to initialize hadoop filesystem for {}.", scheme);
-            org.apache.hadoop.fs.FileSystem azureFS = new AzureBlobFileSystem();
-            azureFS.initialize(fsUri, hadoopConfig);
-            return azureFS;
-        }
+        return new HadoopFileSystem(createAzureFS(fsUri, hadoopConfig));
     }
 }
diff --git a/...stems/flink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/AzureFSFactory.java b/...stems/flink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/AzureFSFactory.java
@@ -18,11 +18,25 @@
 
 package org.apache.flink.fs.azurefs;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+
 /** A factory for the Azure file system over HTTP. */
 public class AzureFSFactory extends AbstractAzureFSFactory {
 
     @Override
     public String getScheme() {
         return "wasb";
     }
+
+    @Override
+    FileSystem createAzureFS(URI fsUri, Configuration hadoopConfig) throws IOException {
+        FileSystem azureFS = new NativeAzureFileSystem();
+        azureFS.initialize(fsUri, hadoopConfig);
+        return azureFS;
+    }
 }
diff --git a/...k-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/SecureABFSAzureFSFactory.java b/...k-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/SecureABFSAzureFSFactory.java
@@ -18,11 +18,25 @@
 
 package org.apache.flink.fs.azurefs;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+
 /** Secure ABFS AzureFS implementation. */
 public class SecureABFSAzureFSFactory extends AbstractAzureFSFactory {
 
     @Override
     public String getScheme() {
         return "abfss";
     }
+
+    @Override
+    FileSystem createAzureFS(URI fsUri, Configuration hadoopConfig) throws IOException {
+        FileSystem azureFS = new AzureBlobFileSystem();
+        azureFS.initialize(fsUri, hadoopConfig);
+        return azureFS;
+    }
 }
diff --git a/...flink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/SecureAzureFSFactory.java b/...flink-azure-fs-hadoop/src/main/java/org/apache/flink/fs/azurefs/SecureAzureFSFactory.java
@@ -18,11 +18,25 @@
 
 package org.apache.flink.fs.azurefs;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+
 /** A factory for the Azure file system over HTTPs. */
 public class SecureAzureFSFactory extends AbstractAzureFSFactory {
 
     @Override
     public String getScheme() {
         return "wasbs";
     }
+
+    @Override
+    FileSystem createAzureFS(URI fsUri, Configuration hadoopConfig) throws IOException {
+        FileSystem azureFS = new NativeAzureFileSystem();
+        azureFS.initialize(fsUri, hadoopConfig);
+        return azureFS;
+    }
 }
diff --git a/flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE
@@ -10,8 +10,6 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.fasterxml.jackson.core:jackson-core:2.12.1
 - com.fasterxml.jackson.core:jackson-databind:2.12.1
 - com.google.guava:guava:20.0
-- com.microsoft.azure:azure-keyvault-core:1.0.0
-- com.microsoft.azure:azure-storage:7.0.1
 - commons-codec:commons-codec:1.13
 - commons-logging:commons-logging:1.1.3
 - org.apache.hadoop:hadoop-azure:3.3.1
@@ -23,3 +21,8 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.eclipse.jetty:jetty-util:9.3.24.v20180605
 - org.eclipse.jetty:jetty-util-ajax:9.3.24.v20180605
 - org.wildfly.openssl:wildfly-openssl:1.0.7.Final
+
+This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
+
+- com.microsoft.azure:azure-keyvault-core:1.0.0
+- com.microsoft.azure:azure-storage:7.0.1
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE
@@ -19,9 +19,6 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.commons:commons-configuration2:2.1.1
 - org.apache.commons:commons-lang3:3.3.2
 - org.apache.commons:commons-text:1.4
-- org.checkerframework:checker-qual:2.5.2
-- org.codehaus.mojo:animal-sniffer-annotations:1.17
-- org.codehaus.woodstox:stax2-api:4.2.1
 - org.xerial.snappy:snappy-java:1.1.8.3
 - commons-lang:commons-lang:2.6
 - commons-collections:commons-collections:3.2.2
@@ -36,6 +33,15 @@ This project bundles the following dependencies under the Apache Software Licens
 - com.fasterxml.jackson.core:jackson-core:2.12.1
 - com.fasterxml.jackson.core:jackson-databind:2.12.1
 - com.fasterxml.woodstox:woodstox-core:5.3.0
+
+This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
+
+- org.checkerframework:checker-qual:2.5.2
+- org.codehaus.mojo:animal-sniffer-annotations:1.17
+
+This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause).
+See bundled license files for details.
+
 - dnsjava:dnsjava:2.1.7
 
 This project bundles the following dependencies under the Go License (https://golang.org/LICENSE).
@@ -46,7 +52,7 @@ See bundled license files for details.
 This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php).
 See bundled license files for details.
 
-- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4)
+- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1)
 
 This project bundles org.apache.hadoop:*:3.3.1 from which it inherits the following notices:
 

diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE
@@ -24,7 +24,6 @@ This project bundles the following dependencies under the Apache Software Licens
 - commons-io:commons-io:2.8.0
 - commons-lang:commons-lang:2.6
 - commons-logging:commons-logging:1.1.3
-- dnsjava:dnsjava:2.1.7
 - joda-time:joda-time:2.5
 - org.apache.commons:commons-configuration2:2.1.1
 - org.apache.commons:commons-lang3:3.3.2
@@ -42,13 +41,20 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.kerby:kerby-pkix:1.0.1
 - org.apache.kerby:kerby-asn1:1.0.1
 - org.apache.kerby:kerby-util:1.0.1
-- org.checkerframework:checker-qual:2.5.2
-- org.codehaus.mojo:animal-sniffer-annotations:1.17
-- org.codehaus.woodstox:stax2-api:4.2.1
 - org.xerial.snappy:snappy-java:1.1.8.3
 - org.wildfly.openssl:wildfly-openssl:1.0.7.Final
 - software.amazon.ion:ion-java:1.0.2
 
+This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause).
+See bundled license files for details.
+
+- dnsjava:dnsjava:2.1.7
+
+This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
+
+- org.checkerframework:checker-qual:2.5.2
+- org.codehaus.mojo:animal-sniffer-annotations:1.17
+
 This project bundles the following dependencies under the CDDL 1.1 license.
 See bundled license files for details.
 
@@ -62,4 +68,4 @@ See bundled license files for details.
 This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php).
 See bundled license files for details.
 
-- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4)
+- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1)
diff --git a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE
@@ -53,13 +53,16 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.kerby:kerb-core:1.0.1
 - org.apache.kerby:kerby-pkix:1.0.1
 - org.apache.kerby:kerby-util:1.0.1
-- org.codehaus.woodstox:stax2-api:4.2.1
 - org.xerial.snappy:snappy-java:1.1.8.3
 - org.weakref:jmxutils:1.19
 - org.wildfly.openssl:wildfly-openssl:1.0.7.Final
-- dnsjava:dnsjava:2.1.7
 - software.amazon.ion:ion-java:1.0.2
 
+This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause).
+See bundled license files for details.
+
+- dnsjava:dnsjava:2.1.7
+
 This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/)
 See bundled license files for details.
 
@@ -78,7 +81,7 @@ See bundled license files for details.
 This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php).
 See bundled license files for details.
 
-- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4)
+- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1)
 
 This project bundles the following dependencies under the Public Domain.
 See bundled license files for details.