improve the transfer config priority (#133)

tencentyun · Jan 22, 2024 · 37ca5e4 · 37ca5e4
1 parent 5cd17f5
commit 37ca5e4
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -350,4 +350,14 @@ fs.cosn.userinfo.appid 12345678
 同理可以简化上一节各Bucket独立配置项通过添加如下配置做到cosn://test/得方式进行访问
 fs.cosn.bucket.test.trsf.fs.ofs.use.short.bucketname true
 fs.cosn.bucket.test.upload.buffer 1024
-fs.cosn.bucket.test.<other config> *
+fs.cosn.bucket.test.<other config> *
+
+### 插件配置优先级说明
+插件配置优先级：Bucket独立配置项大于元数据加速桶配置项（fs.cosn.trsf.*) 大于默认配置。
+
+插件配置转化顺序： Bucket独立配置项-> 显式配置fs.cosn.trsf.* 相关配置则不进行转换-> 没有显式配置fs.cosn.trsf.* 的则会自动转换，例如如下配置项：
+1. fs.cosn.userinfo.appid
+2. fs.cosn.bucket.region
+3. fs.cosn.userinfo.region
+4. fs.cosn.server-side-encryption.algorithm
+5. fs.cosn.server-side-encryption.context
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.qcloud.cos</groupId>
     <artifactId>hadoop-cos</artifactId>
-    <version>8.3.5</version>
+    <version>8.3.6</version>
     <packaging>jar</packaging>
 
     <name>Apache Hadoop Tencent Cloud COS Support</name>

diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java
@@ -482,13 +482,19 @@ private void passThroughRangerConfig() {
         // ofs ranger init get ranger policy auto
         String ofsRangerKey = Constants.COSN_CONFIG_TRANSFER_PREFIX.
                 concat(Constants.COSN_POSIX_BUCKCET_OFS_RANGER_FLAG);
-        if (useOFSRanger()) {
-            // set ofs ranger open
-            this.getConf().setBoolean(ofsRangerKey, true);
-            return;
+        String ofsRangerValue = this.getConf().get(ofsRangerKey);
+        if (null == ofsRangerValue || ofsRangerValue.isEmpty()) {
+            if (useOFSRanger()) {
+                // set ofs ranger open
+                this.getConf().setBoolean(ofsRangerKey, true);
+                return;
+            } else {
+                // set false, avoid sdk change the default value
+                this.getConf().setBoolean(ofsRangerKey, false);
+            }
         } else {
-            // set false, avoid sdk change the default value
-            this.getConf().setBoolean(ofsRangerKey, false);
+            LOG.info("ofs transfer config already exist, transfer key {}, value {}",
+                    ofsRangerKey, ofsRangerValue);
         }
 
         if (!this.rangerCredentialsClient.isEnableRangerPluginPermissionCheck()) {
@@ -499,12 +505,25 @@ private void passThroughRangerConfig() {
         if (this.rangerCredentialsClient.getRangerPolicyUrl() != null) {
             String policyUrlKey = Constants.COSN_CONFIG_TRANSFER_PREFIX.
                     concat(Constants.COSN_POSIX_BUCKET_RANGER_POLICY_URL);
-            this.getConf().set(policyUrlKey, this.rangerCredentialsClient.getRangerPolicyUrl());
+            String policyUrlValue = this.getConf().get(policyUrlKey);
+            if (policyUrlValue == null || policyUrlValue.isEmpty()) {
+                this.getConf().set(policyUrlKey, this.rangerCredentialsClient.getRangerPolicyUrl());
+            } else {
+                LOG.info("ofs transfer config already exist, transfer key {}, value {}",
+                        policyUrlKey, policyUrlValue);
+            }
         }
+
         if (this.rangerCredentialsClient.getAuthJarMd5() != null) {
             String authJarMd5Key = Constants.COSN_CONFIG_TRANSFER_PREFIX.
                     concat(Constants.COSN_POSIX_BUCKET_RANGER_AUTH_JAR_MD5);
-            this.getConf().set(authJarMd5Key, this.rangerCredentialsClient.getAuthJarMd5());
+            String authJarMd5Value = this.getConf().get(authJarMd5Key);
+            if (authJarMd5Value == null || authJarMd5Value.isEmpty()) {
+                this.getConf().set(authJarMd5Key, this.rangerCredentialsClient.getAuthJarMd5());
+            } else {
+                LOG.info("ofs transfer config already exist, transfer key {}, value {}",
+                        authJarMd5Key, authJarMd5Value);
+            }
         }
     }
 
@@ -530,7 +549,7 @@ private void transferOfsConfig() {
                 // if ofs transfer appid set we ignore it, then we can use appid to version control
                 String transferContent = this.getConf().get(transferKey);
                 if (null != transferContent && !transferContent.isEmpty()) {
-                    LOG.info("transfer ofs config, already has transfer key {}, value {}",
+                    LOG.info("ofs transfer config already exist, transfer key {}, value {}",
                             transferKey, transferContent);
                     continue;
                 }