add configuartion to direct use ofs ranger when posix way query (#76)

merge the ranger fix code

remove the checkpermission of append in cosn fs

fix bugs

Co-authored-by: alantong(佟明达) <[email protected]>

src/main/java/org/apache/hadoop/fs/CosFileSystem.java

@@ -18,11 +18,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 
 /**
@@ -47,6 +43,8 @@ public class CosFileSystem extends FileSystem {
     private NativeFileSystemStore nativeStore;
     private boolean isPosixFSStore;
     private boolean isDefaultNativeStore;
+    private boolean isPosixUseOFSRanger;
+    private boolean isPosixImpl = false;
     private FileSystem actualImplFS = null;
 
     private URI uri;
@@ -93,6 +91,9 @@ public void initialize(URI uri, Configuration conf) throws IOException {
             this.isDefaultNativeStore = true;
         }
         this.rangerCredentialsClient = this.nativeStore.getRangerCredentialsClient();
+        this.isPosixUseOFSRanger = this.getConf().
+                getBoolean(CosNConfigKeys.COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED,
+                CosNConfigKeys.DEFAULT_COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED);
 
         // required checkCustomAuth if ranger is enabled and custom authentication is enabled
         checkCustomAuth(conf);
@@ -108,8 +109,8 @@ public void initialize(URI uri, Configuration conf) throws IOException {
                         CosNConfigKeys.DEFAULT_COSN_POSIX_BUCKET_FS_IMPL);
             }
 
-            LOG.info("The posix bucket [{}] use the class [{}] as the filesystem implementation.",
-                    bucket, posixBucketFSImpl);
+            LOG.info("The posix bucket [{}] use the class [{}] as the filesystem implementation, " +
+                            "use each ranger [{}]", bucket, posixBucketFSImpl, this.isPosixUseOFSRanger);
             // if ofs impl.
             // network version start from the 2.7.
             // sdk version start from the 1.0.4.
@@ -121,6 +122,7 @@ public void initialize(URI uri, Configuration conf) throws IOException {
                 ((CosNFileSystem) this.actualImplFS).withStore(this.nativeStore).withBucket(bucket)
                         .withPosixBucket(isPosixFSStore).withRangerCredentialsClient(rangerCredentialsClient);
             } else if (this.actualImplFS instanceof CHDFSHadoopFileSystemAdapter) {
+                this.isPosixImpl = true;
                 // judge whether ranger client contains policy url or other config need to pass to ofs
                 this.passThroughRangerConfig();
                 // before the init, must transfer the config and disable the range in ofs
@@ -358,6 +360,9 @@ public List<String> listXAttrs(Path f) throws IOException {
     public Token<?> getDelegationToken(String renewer) throws IOException {
         LOG.info("getDelegationToken, renewer: {}, stack: {}",
                 renewer, Arrays.toString(Thread.currentThread().getStackTrace()).replace(',', '\n'));
+        if (useOFSRanger()) {
+            return this.actualImplFS.getDelegationToken(renewer);
+        }
         Token<?> token = this.rangerCredentialsClient.doGetDelegationToken(renewer);
         if (token != null)
             return token;
@@ -370,11 +375,23 @@ public NativeFileSystemStore getStore() {
 
     // pass ofs ranger client config to ofs
     private void passThroughRangerConfig() {
+        // ofs ranger init get ranger policy auto
+        String ofsRangerKey = Constants.COSN_CONFIG_TRANSFER_PREFIX.
+                concat(Constants.COSN_POSIX_BUCKCET_OFS_RANGER_FLAG);
+        if (useOFSRanger()) {
+            // set ofs ranger open
+            this.getConf().setBoolean(ofsRangerKey, true);
+            return;
+        } else {
+            // set false, avoid sdk change the default value
+            this.getConf().setBoolean(ofsRangerKey, false);
+        }
+
         if (!this.rangerCredentialsClient.isEnableRangerPluginPermissionCheck()) {
             LOG.info("not enable ranger plugin permission check");
             return;
         }
-        // todo: alantong, ofs java sdk decide the key
+
         if (this.rangerCredentialsClient.getRangerPolicyUrl() != null) {
             String policyUrlKey = Constants.COSN_CONFIG_TRANSFER_PREFIX.
                     concat(Constants.COSN_POSIX_BUCKET_RANGER_POLICY_URL);
@@ -421,18 +438,32 @@ public void releaseFileLock(Path f) throws IOException {
 
     @Override
     public String getCanonicalServiceName() {
+        if (useOFSRanger()) {
+            return this.actualImplFS.getCanonicalServiceName();
+        }
         return this.rangerCredentialsClient.doGetCanonicalServiceName();
     }
 
     private void checkPermission(Path f, RangerAccessType rangerAccessType) throws IOException {
+        if (useOFSRanger()) {
+            return;
+        }
         this.rangerCredentialsClient.doCheckPermission(f, rangerAccessType, getOwnerId(), getWorkingDirectory());
     }
 
+    private boolean useOFSRanger() {
+        if (this.isPosixImpl && this.isPosixUseOFSRanger) {
+            return true;
+        }
+        return false;
+    }
+
     /**
      * @param conf
      * @throws IOException
      */
     private void checkCustomAuth(Configuration conf) throws IOException {
+        // todo: need get token first
         this.rangerCredentialsClient.doCheckCustomAuth(conf);
     }
 

src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java

@@ -155,4 +155,9 @@ public class CosNConfigKeys extends CommonConfigurationKeys {
     public static final boolean DEFAULT_COSN_FLUSH_ENABLED = true;
     public static final String COSN_MAPDISK_DELETEONEXIT_ENABLED = "fs.cosn.map_disk.delete_on_exit.enabled";
     public static final boolean DEFAULT_COSN_MAPDISK_DELETEONEXIT_ENABLED = true;
+
+    // range control, whether meta engine need query own ranger. can be used when transfer from ofs to cos ranger
+    public static final String COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED = "fs.cosn.posix.bucket.use_ofs_ranger.enabled";
+    public static final boolean DEFAULT_COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED = false;
+
 }

src/main/java/org/apache/hadoop/fs/cosn/Constants.java

@@ -41,4 +41,5 @@ private Constants() {
     // posix bucket ranger config need to pass through
     public static final String COSN_POSIX_BUCKET_RANGER_POLICY_URL = "fs.ofs.cosn.ranger.policy.url";
     public static final String COSN_POSIX_BUCKET_RANGER_AUTH_JAR_MD5 = "fs.ofs.cosn.ranger.auth.jar.md5";
+    public static final String COSN_POSIX_BUCKCET_OFS_RANGER_FLAG = "fs.ofs.ranger.enable.flag";
 }