From 1dc60de9c91e8ec6b0c1fee24daba27d0e5d0731 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Tue, 3 Dec 2024 15:51:12 +0800 Subject: [PATCH 01/31] [#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorization.java | 2 + .../ranger/RangerAuthorizationHDFSPlugin.java | 186 ++++++++++++++++++ .../authorization/ranger/RangerHelper.java | 5 +- .../ranger/RangerMetadataObject.java | 4 +- .../ranger/reference/RangerDefines.java | 4 +- 5 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java index 459b6b04720..c873c4a40b2 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java @@ -36,6 +36,8 @@ protected AuthorizationPlugin newPlugin(String catalogProvider, Map config) { + super(config); + } + + public static synchronized RangerAuthorizationHDFSPlugin getInstance(Map config) { + return new RangerAuthorizationHDFSPlugin(config); + } + + @Override + public void validateRangerMetadataObject(List names, RangerMetadataObject.Type type) + throws IllegalArgumentException { + Preconditions.checkArgument( + names != null && !names.isEmpty(), "Cannot create a Ranger metadata object with no names"); + Preconditions.checkArgument( + names.size() != 1, + "Cannot create a Ranger metadata object with the name length which is greater than 3"); + Preconditions.checkArgument( + type != RangerMetadataObject.Type.PATH, + "Cannot create a Ranger metadata object with no type"); + + for (String name : names) { + RangerMetadataObjects.checkName(name); + } + } + + @Override + public Map> privilegesMappingRule() { + return ImmutableMap.of( + Privilege.Name.READ_FILESET, + ImmutableSet.of(RangerPrivileges.RangerHdfsPrivilege.READ), + Privilege.Name.WRITE_FILESET, + ImmutableSet.of(RangerPrivileges.RangerHdfsPrivilege.WRITE)); + } + + @Override + public Set ownerMappingRule() { + return ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE); + } + + @Override + public List policyResourceDefinesRule() { + return ImmutableList.of(RangerDefines.PolicyResource.PATH.getName()); + } + + @Override + public Set allowPrivilegesRule() { + return ImmutableSet.of( + Privilege.Name.CREATE_FILESET, Privilege.Name.READ_FILESET, Privilege.Name.WRITE_FILESET); + } + + @Override + public Set allowMetadataObjectTypesRule() { + return ImmutableSet.of(MetadataObject.Type.FILESET); + } + + @Override + public List translatePrivilege(SecurableObject securableObject) { + List rangerSecurableObjects = new ArrayList<>(); + + securableObject.privileges().stream() + .filter(Objects::nonNull) + .forEach( + gravitinoPrivilege -> { + Set rangerPrivileges = new HashSet<>(); + // Ignore unsupported privileges + if (!privilegesMappingRule().containsKey(gravitinoPrivilege.name())) { + return; + } + privilegesMappingRule().get(gravitinoPrivilege.name()).stream() + .forEach( + rangerPrivilege -> + rangerPrivileges.add( + new RangerPrivileges.RangerHivePrivilegeImpl( + rangerPrivilege, gravitinoPrivilege.condition()))); + + switch (gravitinoPrivilege.name()) { + case CREATE_FILESET: + // Ignore the Gravitino privilege `CREATE_FILESET` in the + // RangerAuthorizationHDFSPlugin + break; + case READ_FILESET: + case WRITE_FILESET: + switch (securableObject.type()) { + case FILESET: + rangerSecurableObjects.add( + generateRangerSecurableObject( + ImmutableList.of(getFileSetPath(securableObject)), + RangerMetadataObject.Type.PATH, + rangerPrivileges)); + break; + default: + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); + } + break; + default: + LOG.warn( + "RangerAuthorizationHDFSPlugin -> privilege {} is not supported for the securable object: {}", + gravitinoPrivilege.name(), + securableObject.type()); + } + }); + + return rangerSecurableObjects; + } + + @Override + public List translateOwner(MetadataObject gravitinoMetadataObject) { + List rangerSecurableObjects = new ArrayList<>(); + + switch (gravitinoMetadataObject.type()) { + case FILESET: + rangerSecurableObjects.add( + generateRangerSecurableObject( + ImmutableList.of(getFileSetPath(gravitinoMetadataObject)), + RangerMetadataObject.Type.PATH, + ownerMappingRule())); + break; + default: + throw new AuthorizationPluginException( + "The owner privilege is not supported for the securable object: %s", + gravitinoMetadataObject.type()); + } + + return rangerSecurableObjects; + } + + @Override + public RangerMetadataObject translateMetadataObject(MetadataObject metadataObject) { + Preconditions.checkArgument( + allowMetadataObjectTypesRule().contains(metadataObject.type()), + String.format( + "The metadata object type %s is not supported in the RangerAuthorizationHDFSPlugin", + metadataObject.type())); + Preconditions.checkArgument( + !(metadataObject instanceof RangerPrivileges), + "The metadata object must be not a RangerPrivileges object."); + List nsMetadataObject = + Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); + Preconditions.checkArgument( + nsMetadataObject.size() > 0, "The metadata object must have at least one name."); + + return new RangerMetadataObjects.RangerMetadataObjectImpl( + null, "location", RangerMetadataObject.Type.PATH); + } + + private String getFileSetPath(MetadataObject metadataObject) { + NameIdentifier identifier = NameIdentifier.of(metadataObject.parent(), metadataObject.name()); + Fileset fileset = GravitinoEnv.getInstance().filesetDispatcher().loadFileset(identifier); + Preconditions.checkArgument( + fileset != null, String.format("Fileset %s is not found", identifier)); + return fileset.storageLocation(); + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java index cb74ad02621..cae1530056e 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java @@ -445,7 +445,10 @@ protected RangerPolicy createPolicyAddResources(RangerMetadataObject metadataObj List nsMetadataObject = metadataObject.names(); for (int i = 0; i < nsMetadataObject.size(); i++) { RangerPolicy.RangerPolicyResource policyResource = - new RangerPolicy.RangerPolicyResource(nsMetadataObject.get(i)); + new RangerPolicy.RangerPolicyResource( + nsMetadataObject.get(i), + false, + metadataObject.type().equals(RangerMetadataObject.Type.PATH)); policy.getResources().put(policyResourceDefines.get(i), policyResource); } return policy; diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java index e6611a17944..72e72fad10c 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java @@ -41,7 +41,9 @@ enum Type { /** A table is mapped the table of relational data sources like Apache Hive, MySQL, etc. */ TABLE(MetadataObject.Type.TABLE), /** A column is a sub-collection of the table that represents a group of same type data. */ - COLUMN(MetadataObject.Type.COLUMN); + COLUMN(MetadataObject.Type.COLUMN), + + PATH(MetadataObject.Type.FILESET); private final MetadataObject.Type metadataType; diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/RangerDefines.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/RangerDefines.java index b81fc3fdc6c..570b0feec61 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/RangerDefines.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/RangerDefines.java @@ -37,8 +37,8 @@ public enum PolicyResource { // In the Ranger 2.4.0 agents-common/src/main/resources/service-defs/ranger-servicedef-hive.json DATABASE("database"), TABLE("table"), - COLUMN("column"); - + COLUMN("column"), + PATH("path"); private final String name; PolicyResource(String name) { From 0bb997bce152cd2a8dfc0b7f3ab6dd4f355c8d53 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Tue, 3 Dec 2024 20:02:58 +0800 Subject: [PATCH 02/31] [#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 30 ++++++++++++++----- .../ranger/RangerAuthorizationPlugin.java | 1 + 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index ffc01cd26d8..dbbeb206ead 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.regex.Pattern; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -26,6 +27,8 @@ public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { private static final Logger LOG = LoggerFactory.getLogger(RangerAuthorizationHDFSPlugin.class); + private static final Pattern pattern = Pattern.compile("^hdfs://[^/]*"); + private RangerAuthorizationHDFSPlugin(Map config) { super(config); } @@ -37,14 +40,15 @@ public static synchronized RangerAuthorizationHDFSPlugin getInstance(Map names, RangerMetadataObject.Type type) throws IllegalArgumentException { + LOG.info("validateRangerMetadataObject {}", names); Preconditions.checkArgument( names != null && !names.isEmpty(), "Cannot create a Ranger metadata object with no names"); Preconditions.checkArgument( - names.size() != 1, - "Cannot create a Ranger metadata object with the name length which is greater than 3"); + names.size() == 1, + "Cannot create a Ranger metadata object with the name length which is not equal 1"); Preconditions.checkArgument( - type != RangerMetadataObject.Type.PATH, - "Cannot create a Ranger metadata object with no type"); + type == RangerMetadataObject.Type.PATH, + String.format("Cannot create a Ranger metadata object with %s type", type)); for (String name : names) { RangerMetadataObjects.checkName(name); @@ -112,6 +116,10 @@ public List translatePrivilege(SecurableObject securableO case READ_FILESET: case WRITE_FILESET: switch (securableObject.type()) { + case METALAKE: + case CATALOG: + case SCHEMA: + break; case FILESET: rangerSecurableObjects.add( generateRangerSecurableObject( @@ -139,8 +147,11 @@ public List translatePrivilege(SecurableObject securableO @Override public List translateOwner(MetadataObject gravitinoMetadataObject) { List rangerSecurableObjects = new ArrayList<>(); - switch (gravitinoMetadataObject.type()) { + case METALAKE: + case CATALOG: + case SCHEMA: + return rangerSecurableObjects; case FILESET: rangerSecurableObjects.add( generateRangerSecurableObject( @@ -177,10 +188,15 @@ public RangerMetadataObject translateMetadataObject(MetadataObject metadataObjec } private String getFileSetPath(MetadataObject metadataObject) { - NameIdentifier identifier = NameIdentifier.of(metadataObject.parent(), metadataObject.name()); + //TODO how to get metalake ? + NameIdentifier identifier = NameIdentifier.parse("metalake." + metadataObject.fullName()); Fileset fileset = GravitinoEnv.getInstance().filesetDispatcher().loadFileset(identifier); Preconditions.checkArgument( fileset != null, String.format("Fileset %s is not found", identifier)); - return fileset.storageLocation(); + String filesetLocation = fileset.storageLocation(); + LOG.warn("getFileSetPath filesetLocation {}", filesetLocation); + Preconditions.checkArgument( + filesetLocation != null, String.format("Fileset %s location is not found", identifier)); + return pattern.matcher(filesetLocation).replaceAll(""); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 1b2c924d262..f9cc869d0bc 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -375,6 +375,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n break; case SCHEMA: case TABLE: + case FILESET: // The schema and table use user/group to manage the owner rangerSecurableObjects.stream() .forEach( From 8a212a6c9a342e47ab476a0d6e707a5092c78e27 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 4 Dec 2024 15:44:33 +0800 Subject: [PATCH 03/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 52 ++++++++----------- .../ranger/RangerMetadataObject.java | 4 +- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index dbbeb206ead..525a537c1a4 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -15,6 +15,9 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.authorization.AuthorizationMetadataObject; +import org.apache.gravitino.authorization.AuthorizationPrivilege; +import org.apache.gravitino.authorization.AuthorizationSecurableObject; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; @@ -38,25 +41,7 @@ public static synchronized RangerAuthorizationHDFSPlugin getInstance(Map names, RangerMetadataObject.Type type) - throws IllegalArgumentException { - LOG.info("validateRangerMetadataObject {}", names); - Preconditions.checkArgument( - names != null && !names.isEmpty(), "Cannot create a Ranger metadata object with no names"); - Preconditions.checkArgument( - names.size() == 1, - "Cannot create a Ranger metadata object with the name length which is not equal 1"); - Preconditions.checkArgument( - type == RangerMetadataObject.Type.PATH, - String.format("Cannot create a Ranger metadata object with %s type", type)); - - for (String name : names) { - RangerMetadataObjects.checkName(name); - } - } - - @Override - public Map> privilegesMappingRule() { + public Map> privilegesMappingRule() { return ImmutableMap.of( Privilege.Name.READ_FILESET, ImmutableSet.of(RangerPrivileges.RangerHdfsPrivilege.READ), @@ -65,7 +50,7 @@ public Map> privilegesMappingRule() { } @Override - public Set ownerMappingRule() { + public Set ownerMappingRule() { return ImmutableSet.of( RangerPrivileges.RangerHdfsPrivilege.READ, RangerPrivileges.RangerHdfsPrivilege.WRITE, @@ -89,14 +74,14 @@ public Set allowMetadataObjectTypesRule() { } @Override - public List translatePrivilege(SecurableObject securableObject) { - List rangerSecurableObjects = new ArrayList<>(); + public List translatePrivilege(SecurableObject securableObject) { + List rangerSecurableObjects = new ArrayList<>(); securableObject.privileges().stream() .filter(Objects::nonNull) .forEach( gravitinoPrivilege -> { - Set rangerPrivileges = new HashSet<>(); + Set rangerPrivileges = new HashSet<>(); // Ignore unsupported privileges if (!privilegesMappingRule().containsKey(gravitinoPrivilege.name())) { return; @@ -122,7 +107,7 @@ public List translatePrivilege(SecurableObject securableO break; case FILESET: rangerSecurableObjects.add( - generateRangerSecurableObject( + generateAuthorizationSecurableObject( ImmutableList.of(getFileSetPath(securableObject)), RangerMetadataObject.Type.PATH, rangerPrivileges)); @@ -145,8 +130,8 @@ public List translatePrivilege(SecurableObject securableO } @Override - public List translateOwner(MetadataObject gravitinoMetadataObject) { - List rangerSecurableObjects = new ArrayList<>(); + public List translateOwner(MetadataObject gravitinoMetadataObject) { + List rangerSecurableObjects = new ArrayList<>(); switch (gravitinoMetadataObject.type()) { case METALAKE: case CATALOG: @@ -154,7 +139,7 @@ public List translateOwner(MetadataObject gravitinoMetada return rangerSecurableObjects; case FILESET: rangerSecurableObjects.add( - generateRangerSecurableObject( + generateAuthorizationSecurableObject( ImmutableList.of(getFileSetPath(gravitinoMetadataObject)), RangerMetadataObject.Type.PATH, ownerMappingRule())); @@ -169,7 +154,7 @@ public List translateOwner(MetadataObject gravitinoMetada } @Override - public RangerMetadataObject translateMetadataObject(MetadataObject metadataObject) { + public AuthorizationMetadataObject translateMetadataObject(MetadataObject metadataObject) { Preconditions.checkArgument( allowMetadataObjectTypesRule().contains(metadataObject.type()), String.format( @@ -183,12 +168,17 @@ public RangerMetadataObject translateMetadataObject(MetadataObject metadataObjec Preconditions.checkArgument( nsMetadataObject.size() > 0, "The metadata object must have at least one name."); - return new RangerMetadataObjects.RangerMetadataObjectImpl( - null, "location", RangerMetadataObject.Type.PATH); + RangerMetadataObject rangerMetadataObject = + new RangerMetadataObject( + AuthorizationMetadataObject.getParentFullName(nsMetadataObject), + AuthorizationMetadataObject.getLastName(nsMetadataObject), + RangerMetadataObject.Type.PATH); + rangerMetadataObject.validateAuthorizationMetadataObject(); + return rangerMetadataObject; } private String getFileSetPath(MetadataObject metadataObject) { - //TODO how to get metalake ? + // TODO how to get metalake ? NameIdentifier identifier = NameIdentifier.parse("metalake." + metadataObject.fullName()); Fileset fileset = GravitinoEnv.getInstance().filesetDispatcher().loadFileset(identifier); Preconditions.checkArgument( diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java index 79f58e16eca..2e039b6d9c5 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java @@ -116,8 +116,8 @@ public void validateAuthorizationMetadataObject() throws IllegalArgumentExceptio "If the length of names is 1, it must be the SCHEMA type"); Preconditions.checkArgument( - names.size() != 2 || type == RangerMetadataObject.Type.TABLE, - "If the length of names is 2, it must be the TABLE type"); + names.size() != 2 || type == RangerMetadataObject.Type.TABLE || type == Type.PATH, + "If the length of names is 2, it must be the TABLE type of PATH type"); Preconditions.checkArgument( names.size() != 3 || type == RangerMetadataObject.Type.COLUMN, From 2e8283dec67a0aad13dcf8aa4b76193555a3fbcb Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 4 Dec 2024 16:00:24 +0800 Subject: [PATCH 04/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 22 ++++++++++++++++++- .../ranger/RangerMetadataObject.java | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 525a537c1a4..1fb61ba5985 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.gravitino.authorization.ranger; import com.google.common.base.Preconditions; @@ -46,7 +64,9 @@ public Map> privilegesMappingRule() Privilege.Name.READ_FILESET, ImmutableSet.of(RangerPrivileges.RangerHdfsPrivilege.READ), Privilege.Name.WRITE_FILESET, - ImmutableSet.of(RangerPrivileges.RangerHdfsPrivilege.WRITE)); + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE)); } @Override diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java index 2e039b6d9c5..49205e0149d 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java @@ -36,7 +36,7 @@ public enum Type implements AuthorizationMetadataObject.Type { TABLE(MetadataObject.Type.TABLE), /** A column is a sub-collection of the table that represents a group of same type data. */ COLUMN(MetadataObject.Type.COLUMN), - + /** A path is mapped the path of storages like HDFS, S3 etc. */ PATH(MetadataObject.Type.FILESET); private final MetadataObject.Type metadataType; From 2b021ab18ced1e35e283688fc8e510c3b21b7c41 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 4 Dec 2024 16:40:31 +0800 Subject: [PATCH 05/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 12 ++++ .../RangerAuthorizationHadoopSQLPlugin.java | 15 +++++ .../ranger/RangerAuthorizationPlugin.java | 60 +++++++++++++++++-- .../authorization/ranger/RangerHelper.java | 58 ------------------ 4 files changed, 81 insertions(+), 64 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 1fb61ba5985..3423d6adfe9 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -42,6 +42,7 @@ import org.apache.gravitino.authorization.ranger.reference.RangerDefines; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.file.Fileset; +import org.apache.ranger.plugin.model.RangerPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,6 +83,17 @@ public List policyResourceDefinesRule() { return ImmutableList.of(RangerDefines.PolicyResource.PATH.getName()); } + @Override + protected RangerPolicy createPolicyAddResources(AuthorizationMetadataObject metadataObject) { + RangerPolicy policy = new RangerPolicy(); + policy.setService(rangerServiceName); + policy.setName(metadataObject.fullName()); + RangerPolicy.RangerPolicyResource policyResource = + new RangerPolicy.RangerPolicyResource(metadataObject.names().get(0), false, true); + policy.getResources().put(RangerDefines.PolicyResource.PATH.getName(), policyResource); + return policy; + } + @Override public Set allowPrivilegesRule() { return ImmutableSet.of( diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java index d403d446993..54a90ca1d59 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java @@ -41,6 +41,7 @@ import org.apache.gravitino.authorization.ranger.RangerPrivileges.RangerHadoopSQLPrivilege; import org.apache.gravitino.authorization.ranger.reference.RangerDefines.PolicyResource; import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.apache.ranger.plugin.model.RangerPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,6 +104,20 @@ public List policyResourceDefinesRule() { PolicyResource.COLUMN.getName()); } + @Override + protected RangerPolicy createPolicyAddResources(AuthorizationMetadataObject metadataObject) { + RangerPolicy policy = new RangerPolicy(); + policy.setService(rangerServiceName); + policy.setName(metadataObject.fullName()); + List nsMetadataObject = metadataObject.names(); + for (int i = 0; i < nsMetadataObject.size(); i++) { + RangerPolicy.RangerPolicyResource policyResource = + new RangerPolicy.RangerPolicyResource(nsMetadataObject.get(i)); + policy.getResources().put(policyResourceDefinesRule().get(i), policyResource); + } + return policy; + } + @Override /** Allow privilege operation defines rule. */ public Set allowPrivilegesRule() { diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 3948bc72875..e8e6cdba8b8 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -115,6 +115,57 @@ protected RangerAuthorizationPlugin(Map config) { */ public abstract List policyResourceDefinesRule(); + /** + * Create a new policy for metadata object + * + * @return The RangerPolicy for metadata object. + */ + protected abstract RangerPolicy createPolicyAddResources( + AuthorizationMetadataObject metadataObject); + + protected RangerPolicy addOwnerToNewPolicy( + AuthorizationMetadataObject metadataObject, Owner newOwner) { + RangerPolicy policy = createPolicyAddResources(metadataObject); + ownerMappingRule() + .forEach( + ownerPrivilege -> { + // Each owner's privilege will create one RangerPolicyItemAccess in the policy + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem + .getAccesses() + .add(new RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName())); + if (newOwner != null) { + if (newOwner.type() == Owner.Type.USER) { + policyItem.getUsers().add(newOwner.name()); + } else { + policyItem.getGroups().add(newOwner.name()); + } + // mark the policy item is created by Gravitino + policyItem.getRoles().add(RangerHelper.GRAVITINO_OWNER_ROLE); + } + policy.getPolicyItems().add(policyItem); + }); + return policy; + } + + protected RangerPolicy addOwnerRoleToNewPolicy( + AuthorizationMetadataObject metadataObject, String ownerRoleName) { + RangerPolicy policy = createPolicyAddResources(metadataObject); + + ownerMappingRule() + .forEach( + ownerPrivilege -> { + // Each owner's privilege will create one RangerPolicyItemAccess in the policy + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem + .getAccesses() + .add(new RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName())); + policyItem.getRoles().add(rangerHelper.generateGravitinoRoleName(ownerRoleName)); + policy.getPolicyItems().add(policyItem); + }); + return policy; + } + /** * Create a new role in the Ranger.
* 1. Create a policy for metadata object.
@@ -374,9 +425,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n rangerHelper.findManagedPolicy(AuthorizationSecurableObject); try { if (policy == null) { - policy = - rangerHelper.addOwnerRoleToNewPolicy( - AuthorizationSecurableObject, ownerRoleName); + policy = addOwnerRoleToNewPolicy(AuthorizationSecurableObject, ownerRoleName); rangerClient.createPolicy(policy); } else { rangerHelper.updatePolicyOwnerRole(policy, ownerRoleName); @@ -399,8 +448,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n rangerHelper.findManagedPolicy(AuthorizationSecurableObject); try { if (policy == null) { - policy = - rangerHelper.addOwnerToNewPolicy(AuthorizationSecurableObject, newOwner); + policy = addOwnerToNewPolicy(AuthorizationSecurableObject, newOwner); rangerClient.createPolicy(policy); } else { rangerHelper.updatePolicyOwner(policy, preOwner, newOwner); @@ -674,7 +722,7 @@ private boolean doAddSecurableObject( return true; } } else { - policy = rangerHelper.createPolicyAddResources(securableObject); + policy = createPolicyAddResources(securableObject); } rangerHelper.addPolicyItem(policy, roleName, securableObject); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java index 168922de4d9..4c2b2956c8c 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java @@ -442,64 +442,6 @@ protected void updatePolicyOwner(RangerPolicy policy, Owner preOwner, Owner newO }); } - protected RangerPolicy createPolicyAddResources(AuthorizationMetadataObject metadataObject) { - RangerPolicy policy = new RangerPolicy(); - policy.setService(rangerServiceName); - policy.setName(metadataObject.fullName()); - List nsMetadataObject = metadataObject.names(); - for (int i = 0; i < nsMetadataObject.size(); i++) { - RangerPolicy.RangerPolicyResource policyResource = - new RangerPolicy.RangerPolicyResource( - nsMetadataObject.get(i), - false, - metadataObject.type().equals(RangerMetadataObject.Type.PATH)); - policy.getResources().put(policyResourceDefines.get(i), policyResource); - } - return policy; - } - - protected RangerPolicy addOwnerToNewPolicy( - AuthorizationMetadataObject metadataObject, Owner newOwner) { - RangerPolicy policy = createPolicyAddResources(metadataObject); - - ownerPrivileges.forEach( - ownerPrivilege -> { - // Each owner's privilege will create one RangerPolicyItemAccess in the policy - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); - policyItem - .getAccesses() - .add(new RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName())); - if (newOwner != null) { - if (newOwner.type() == Owner.Type.USER) { - policyItem.getUsers().add(newOwner.name()); - } else { - policyItem.getGroups().add(newOwner.name()); - } - // mark the policy item is created by Gravitino - policyItem.getRoles().add(GRAVITINO_OWNER_ROLE); - } - policy.getPolicyItems().add(policyItem); - }); - return policy; - } - - protected RangerPolicy addOwnerRoleToNewPolicy( - AuthorizationMetadataObject metadataObject, String ownerRoleName) { - RangerPolicy policy = createPolicyAddResources(metadataObject); - - ownerPrivileges.forEach( - ownerPrivilege -> { - // Each owner's privilege will create one RangerPolicyItemAccess in the policy - RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); - policyItem - .getAccesses() - .add(new RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName())); - policyItem.getRoles().add(generateGravitinoRoleName(ownerRoleName)); - policy.getPolicyItems().add(policyItem); - }); - return policy; - } - protected void updatePolicyOwnerRole(RangerPolicy policy, String ownerRoleName) { // Find matching policy items based on the owner's privileges List matchPolicyItems = From adbfc53684fe804f4535e23bcb0ebfebe4a78506 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 4 Dec 2024 16:45:03 +0800 Subject: [PATCH 06/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../authorization/ranger/RangerAuthorizationHDFSPlugin.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 3423d6adfe9..ae4563e4a28 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -192,9 +192,6 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada String.format( "The metadata object type %s is not supported in the RangerAuthorizationHDFSPlugin", metadataObject.type())); - Preconditions.checkArgument( - !(metadataObject instanceof RangerPrivileges), - "The metadata object must be not a RangerPrivileges object."); List nsMetadataObject = Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); Preconditions.checkArgument( From 4991698b428ee23bce5fe960639a898de33282b5 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 4 Dec 2024 18:11:56 +0800 Subject: [PATCH 07/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../authorization/ranger/RangerAuthorization.java | 2 +- .../ranger/RangerAuthorizationHDFSPlugin.java | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java index 2ff0d63a960..04c40e219ef 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java @@ -38,7 +38,7 @@ protected AuthorizationPlugin newPlugin( case "lakehouse-paimon": return RangerAuthorizationHadoopSQLPlugin.getInstance(metalake, config); case "hadoop": - return RangerAuthorizationHDFSPlugin.getInstance(config); + return RangerAuthorizationHDFSPlugin.getInstance(metalake, config); default: throw new IllegalArgumentException("Unknown catalog provider: " + catalogProvider); } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index ae4563e4a28..6493034bc2b 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -51,12 +51,13 @@ public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { private static final Pattern pattern = Pattern.compile("^hdfs://[^/]*"); - private RangerAuthorizationHDFSPlugin(Map config) { - super(config); + private RangerAuthorizationHDFSPlugin(String metalake, Map config) { + super(metalake, config); } - public static synchronized RangerAuthorizationHDFSPlugin getInstance(Map config) { - return new RangerAuthorizationHDFSPlugin(config); + public static synchronized RangerAuthorizationHDFSPlugin getInstance( + String metalake, Map config) { + return new RangerAuthorizationHDFSPlugin(metalake, config); } @Override @@ -207,8 +208,8 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada } private String getFileSetPath(MetadataObject metadataObject) { - // TODO how to get metalake ? - NameIdentifier identifier = NameIdentifier.parse("metalake." + metadataObject.fullName()); + NameIdentifier identifier = + NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); Fileset fileset = GravitinoEnv.getInstance().filesetDispatcher().loadFileset(identifier); Preconditions.checkArgument( fileset != null, String.format("Fileset %s is not found", identifier)); From 8198437b05dc7b8605d4f16ca2fa572945da32b2 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 4 Dec 2024 18:58:55 +0800 Subject: [PATCH 08/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 6493034bc2b..e1f89a7368d 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -51,15 +51,24 @@ public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { private static final Pattern pattern = Pattern.compile("^hdfs://[^/]*"); + private static volatile RangerAuthorizationHDFSPlugin instance = null; private RangerAuthorizationHDFSPlugin(String metalake, Map config) { super(metalake, config); } public static synchronized RangerAuthorizationHDFSPlugin getInstance( - String metalake, Map config) { - return new RangerAuthorizationHDFSPlugin(metalake, config); + String metalake, Map config) { + if (instance == null) { + synchronized (RangerAuthorizationHadoopSQLPlugin.class) { + if (instance == null) { + instance = new RangerAuthorizationHDFSPlugin(metalake, config); + } + } + } + return instance; } + @Override public Map> privilegesMappingRule() { return ImmutableMap.of( From cd0613e2863125e2ec203988b0b3608d3248945e Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 5 Dec 2024 10:18:00 +0800 Subject: [PATCH 09/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../authorization/ranger/RangerAuthorizationHDFSPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index e1f89a7368d..4a0fd2698d3 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -52,12 +52,13 @@ public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { private static final Pattern pattern = Pattern.compile("^hdfs://[^/]*"); private static volatile RangerAuthorizationHDFSPlugin instance = null; + private RangerAuthorizationHDFSPlugin(String metalake, Map config) { super(metalake, config); } public static synchronized RangerAuthorizationHDFSPlugin getInstance( - String metalake, Map config) { + String metalake, Map config) { if (instance == null) { synchronized (RangerAuthorizationHadoopSQLPlugin.class) { if (instance == null) { @@ -68,7 +69,6 @@ public static synchronized RangerAuthorizationHDFSPlugin getInstance( return instance; } - @Override public Map> privilegesMappingRule() { return ImmutableMap.of( From 2882c37eae29d6f854bcd317510953234510d1e8 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 5 Dec 2024 16:47:09 +0800 Subject: [PATCH 10/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 5 +- .../test/RangerAuthorizationHDFSPluginIT.java | 152 ++++++++++++++++++ .../ranger/integration/test/RangerITEnv.java | 31 ++++ 3 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 4a0fd2698d3..55200daac78 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -217,13 +217,16 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada } private String getFileSetPath(MetadataObject metadataObject) { + boolean testEnv = System.getenv("GRAVITINO_TEST") != null; + if (testEnv) { + return metadataObject.fullName(); + } NameIdentifier identifier = NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); Fileset fileset = GravitinoEnv.getInstance().filesetDispatcher().loadFileset(identifier); Preconditions.checkArgument( fileset != null, String.format("Fileset %s is not found", identifier)); String filesetLocation = fileset.storageLocation(); - LOG.warn("getFileSetPath filesetLocation {}", filesetLocation); Preconditions.checkArgument( filesetLocation != null, String.format("Fileset %s location is not found", identifier)); return pattern.matcher(filesetLocation).replaceAll(""); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java new file mode 100644 index 00000000000..3fa905cc5e2 --- /dev/null +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -0,0 +1,152 @@ +package org.apache.gravitino.authorization.ranger.integration.test; + +import com.google.common.collect.Lists; +import java.util.List; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.authorization.AuthorizationMetadataObject; +import org.apache.gravitino.authorization.AuthorizationSecurableObject; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; +import org.apache.gravitino.authorization.ranger.RangerMetadataObject; +import org.apache.gravitino.authorization.ranger.RangerPrivileges; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("gravitino-docker-test") +public class RangerAuthorizationHDFSPluginIT { + + private static RangerAuthorizationPlugin rangerAuthPlugin; + + @BeforeAll + public static void setup() { + RangerITEnv.init(); + rangerAuthPlugin = RangerITEnv.rangerAuthHDFSPlugin; + } + + @Test + public void testTranslateMetadataObject() { + MetadataObject metalake = + MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); + Assertions.assertThrows( + IllegalArgumentException.class, () -> rangerAuthPlugin.translateMetadataObject(metalake)); + + MetadataObject catalog = + MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); + Assertions.assertThrows( + IllegalArgumentException.class, () -> rangerAuthPlugin.translateMetadataObject(catalog)); + + MetadataObject schema = + MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); + Assertions.assertThrows( + IllegalArgumentException.class, () -> rangerAuthPlugin.translateMetadataObject(schema)); + + MetadataObject table = + MetadataObjects.parse(String.format("catalog1.schema1.tab1"), MetadataObject.Type.TABLE); + Assertions.assertThrows( + IllegalArgumentException.class, () -> rangerAuthPlugin.translateMetadataObject(table)); + + MetadataObject fileset = + MetadataObjects.parse( + String.format("catalog1.schema1.fileset1"), MetadataObject.Type.FILESET); + AuthorizationMetadataObject rangerFileset = rangerAuthPlugin.translateMetadataObject(fileset); + Assertions.assertEquals(2, rangerFileset.names().size()); + Assertions.assertEquals("schema1", rangerFileset.names().get(0)); + Assertions.assertEquals("fileset1", rangerFileset.names().get(1)); + Assertions.assertEquals(RangerMetadataObject.Type.TABLE, rangerFileset.type()); + } + + @Test + public void testTranslatePrivilege() { + SecurableObject filesetInMetalake = + SecurableObjects.parse( + String.format("metalake1"), + MetadataObject.Type.METALAKE, + Lists.newArrayList( + Privileges.CreateFileset.allow(), + Privileges.ReadFileset.allow(), + Privileges.WriteFileset.allow())); + List filesetInMetalake1 = + rangerAuthPlugin.translatePrivilege(filesetInMetalake); + Assertions.assertEquals(0, filesetInMetalake1.size()); + + SecurableObject filesetInCatalog = + SecurableObjects.parse( + String.format("catalog1"), + MetadataObject.Type.CATALOG, + Lists.newArrayList( + Privileges.CreateFileset.allow(), + Privileges.ReadFileset.allow(), + Privileges.WriteFileset.allow())); + List filesetInCatalog1 = + rangerAuthPlugin.translatePrivilege(filesetInCatalog); + Assertions.assertEquals(0, filesetInCatalog1.size()); + + SecurableObject filesetInSchema = + SecurableObjects.parse( + String.format("catalog1.schema1"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList( + Privileges.CreateFileset.allow(), + Privileges.ReadFileset.allow(), + Privileges.WriteFileset.allow())); + List filesetInSchema1 = + rangerAuthPlugin.translatePrivilege(filesetInSchema); + Assertions.assertEquals(0, filesetInSchema1.size()); + + SecurableObject filesetInFileset = + SecurableObjects.parse( + String.format("catalog1.schema1.fileset1"), + MetadataObject.Type.FILESET, + Lists.newArrayList( + Privileges.CreateFileset.allow(), + Privileges.ReadFileset.allow(), + Privileges.WriteFileset.allow())); + List filesetInFileset1 = + rangerAuthPlugin.translatePrivilege(filesetInFileset); + Assertions.assertEquals(2, filesetInSchema1.size()); + Assertions.assertEquals("catalog1.schema1.fileset1", filesetInFileset1.get(0).fullName()); + Assertions.assertEquals(RangerMetadataObject.Type.PATH, filesetInFileset1.get(0).type()); + filesetInFileset1 + .get(0) + .privileges() + .forEach( + privilege -> + Assertions.assertEquals( + RangerPrivileges.RangerHdfsPrivilege.READ.getName(), privilege.getName())); + Assertions.assertEquals("catalog1.schema1.fileset1", filesetInFileset1.get(1).fullName()); + Assertions.assertEquals(RangerMetadataObject.Type.PATH, filesetInFileset1.get(1).type()); + Assertions.assertEquals(2, filesetInFileset1.get(1).privileges().size()); + } + + @Test + public void testTranslateOwner() { + MetadataObject metalake = + MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); + List metalakeOwner = rangerAuthPlugin.translateOwner(metalake); + Assertions.assertEquals(0, metalakeOwner.size()); + + MetadataObject catalog = + MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); + List catalogOwner = rangerAuthPlugin.translateOwner(catalog); + Assertions.assertEquals(0, catalogOwner.size()); + + MetadataObject schema = + MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); + List schemaOwner = rangerAuthPlugin.translateOwner(schema); + Assertions.assertEquals(0, schemaOwner.size()); + + MetadataObject fileset = + MetadataObjects.parse( + String.format("catalog1.schema1.fileset1"), MetadataObject.Type.FILESET); + List filesetOwner = rangerAuthPlugin.translateOwner(fileset); + Assertions.assertEquals(1, filesetOwner.size()); + Assertions.assertEquals("catalog1.schema1.fileset1", filesetOwner.get(0).fullName()); + Assertions.assertEquals(RangerMetadataObject.Type.PATH, filesetOwner.get(0).type()); + Assertions.assertEquals(3, filesetOwner.get(0).privileges().size()); + } +} diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 2758d307bad..46703cf1b7b 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -30,6 +30,7 @@ import org.apache.gravitino.authorization.AuthorizationSecurableObject; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.ranger.RangerAuthorizationHDFSPlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationHadoopSQLPlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.authorization.ranger.RangerHelper; @@ -81,8 +82,11 @@ public class RangerITEnv { // Search filter prefix file path constants public static final String SEARCH_FILTER_PATH = SearchFilter.RESOURCE_PREFIX + RESOURCE_PATH; public static RangerAuthorizationPlugin rangerAuthHivePlugin; + public static RangerAuthorizationPlugin rangerAuthHDFSPlugin; protected static RangerHelper rangerHelper; + protected static RangerHelper rangerHDFSHelper; + public static void init() { containerSuite.startRangerContainer(); rangerClient = containerSuite.getRangerContainer().rangerClient; @@ -104,6 +108,25 @@ public static void init() { RangerContainer.rangerPassword, AuthorizationPropertiesMeta.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME)); + + rangerAuthHDFSPlugin = + RangerAuthorizationHDFSPlugin.getInstance( + "metalake", + ImmutableMap.of( + AuthorizationPropertiesMeta.RANGER_ADMIN_URL, + String.format( + "http://%s:%d", + containerSuite.getRangerContainer().getContainerIpAddress(), + RangerContainer.RANGER_SERVER_PORT), + AuthorizationPropertiesMeta.RANGER_AUTH_TYPE, + RangerContainer.authType, + AuthorizationPropertiesMeta.RANGER_USERNAME, + RangerContainer.rangerUserName, + AuthorizationPropertiesMeta.RANGER_PASSWORD, + RangerContainer.rangerPassword, + AuthorizationPropertiesMeta.RANGER_SERVICE_NAME, + RangerITEnv.RANGER_HDFS_REPO_NAME)); + rangerHelper = new RangerHelper( rangerClient, @@ -112,6 +135,14 @@ public static void init() { rangerAuthHivePlugin.ownerMappingRule(), rangerAuthHivePlugin.policyResourceDefinesRule()); + rangerHDFSHelper = + new RangerHelper( + rangerClient, + RangerContainer.rangerUserName, + RangerITEnv.RANGER_HDFS_REPO_NAME, + rangerAuthHDFSPlugin.ownerMappingRule(), + rangerAuthHDFSPlugin.policyResourceDefinesRule()); + if (!initRangerService) { synchronized (RangerITEnv.class) { // No IP address set, no impact on testing From 08e0513998d274de564eb020764389c9ee6335c4 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 5 Dec 2024 17:35:15 +0800 Subject: [PATCH 11/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 11 ++--- .../ranger/RangerMetadataObject.java | 8 ++-- .../test/RangerAuthorizationHDFSPluginIT.java | 48 ++++++++++++------- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 55200daac78..e7c48ede3fe 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -150,7 +150,7 @@ public List translatePrivilege(SecurableObject sec case FILESET: rangerSecurableObjects.add( generateAuthorizationSecurableObject( - ImmutableList.of(getFileSetPath(securableObject)), + translateMetadataObject(securableObject).names(), RangerMetadataObject.Type.PATH, rangerPrivileges)); break; @@ -182,7 +182,7 @@ public List translateOwner(MetadataObject gravitin case FILESET: rangerSecurableObjects.add( generateAuthorizationSecurableObject( - ImmutableList.of(getFileSetPath(gravitinoMetadataObject)), + translateMetadataObject(gravitinoMetadataObject).names(), RangerMetadataObject.Type.PATH, ownerMappingRule())); break; @@ -207,11 +207,10 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada Preconditions.checkArgument( nsMetadataObject.size() > 0, "The metadata object must have at least one name."); + nsMetadataObject.remove(0); // Remove the catalog name RangerMetadataObject rangerMetadataObject = new RangerMetadataObject( - AuthorizationMetadataObject.getParentFullName(nsMetadataObject), - AuthorizationMetadataObject.getLastName(nsMetadataObject), - RangerMetadataObject.Type.PATH); + null, getFileSetPath(metadataObject), RangerMetadataObject.Type.PATH); rangerMetadataObject.validateAuthorizationMetadataObject(); return rangerMetadataObject; } @@ -219,7 +218,7 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada private String getFileSetPath(MetadataObject metadataObject) { boolean testEnv = System.getenv("GRAVITINO_TEST") != null; if (testEnv) { - return metadataObject.fullName(); + return "/test"; } NameIdentifier identifier = NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java index 49205e0149d..3378c65e3d1 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java @@ -112,12 +112,12 @@ public void validateAuthorizationMetadataObject() throws IllegalArgumentExceptio type != null, "Cannot create a Ranger metadata object with no type"); Preconditions.checkArgument( - names.size() != 1 || type == RangerMetadataObject.Type.SCHEMA, - "If the length of names is 1, it must be the SCHEMA type"); + names.size() != 1 || type == RangerMetadataObject.Type.SCHEMA || type == Type.PATH, + "If the length of names is 1, it must be the SCHEMA type of PATH type"); Preconditions.checkArgument( - names.size() != 2 || type == RangerMetadataObject.Type.TABLE || type == Type.PATH, - "If the length of names is 2, it must be the TABLE type of PATH type"); + names.size() != 2 || type == RangerMetadataObject.Type.TABLE, + "If the length of names is 2, it must be the TABLE type"); Preconditions.checkArgument( names.size() != 3 || type == RangerMetadataObject.Type.COLUMN, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index 3fa905cc5e2..f4bd0bc6423 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -1,5 +1,6 @@ package org.apache.gravitino.authorization.ranger.integration.test; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.List; import org.apache.gravitino.MetadataObject; @@ -54,10 +55,9 @@ public void testTranslateMetadataObject() { MetadataObjects.parse( String.format("catalog1.schema1.fileset1"), MetadataObject.Type.FILESET); AuthorizationMetadataObject rangerFileset = rangerAuthPlugin.translateMetadataObject(fileset); - Assertions.assertEquals(2, rangerFileset.names().size()); - Assertions.assertEquals("schema1", rangerFileset.names().get(0)); - Assertions.assertEquals("fileset1", rangerFileset.names().get(1)); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, rangerFileset.type()); + Assertions.assertEquals(1, rangerFileset.names().size()); + Assertions.assertEquals("/test", rangerFileset.fullName()); + Assertions.assertEquals(RangerMetadataObject.Type.PATH, rangerFileset.type()); } @Test @@ -108,19 +108,31 @@ public void testTranslatePrivilege() { Privileges.WriteFileset.allow())); List filesetInFileset1 = rangerAuthPlugin.translatePrivilege(filesetInFileset); - Assertions.assertEquals(2, filesetInSchema1.size()); - Assertions.assertEquals("catalog1.schema1.fileset1", filesetInFileset1.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.PATH, filesetInFileset1.get(0).type()); - filesetInFileset1 - .get(0) - .privileges() - .forEach( - privilege -> - Assertions.assertEquals( - RangerPrivileges.RangerHdfsPrivilege.READ.getName(), privilege.getName())); - Assertions.assertEquals("catalog1.schema1.fileset1", filesetInFileset1.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.PATH, filesetInFileset1.get(1).type()); - Assertions.assertEquals(2, filesetInFileset1.get(1).privileges().size()); + Assertions.assertEquals(2, filesetInFileset1.size()); + + filesetInFileset1.forEach( + securableObject -> { + Assertions.assertEquals(RangerMetadataObject.Type.PATH, securableObject.type()); + Assertions.assertEquals("/test", securableObject.fullName()); + Assertions.assertTrue( + securableObject.privileges().size() == 1 || securableObject.privileges().size() == 2); + if (securableObject.privileges().size() == 1) { + Assertions.assertEquals( + RangerPrivileges.RangerHdfsPrivilege.READ.getName(), + securableObject.privileges().get(0).getName()); + } else { + securableObject + .privileges() + .forEach( + privilege -> { + Assertions.assertTrue( + ImmutableList.of( + RangerPrivileges.RangerHdfsPrivilege.WRITE.getName(), + RangerPrivileges.RangerHdfsPrivilege.EXECUTE.getName()) + .contains(privilege.getName())); + }); + } + }); } @Test @@ -145,7 +157,7 @@ public void testTranslateOwner() { String.format("catalog1.schema1.fileset1"), MetadataObject.Type.FILESET); List filesetOwner = rangerAuthPlugin.translateOwner(fileset); Assertions.assertEquals(1, filesetOwner.size()); - Assertions.assertEquals("catalog1.schema1.fileset1", filesetOwner.get(0).fullName()); + Assertions.assertEquals("/test", filesetOwner.get(0).fullName()); Assertions.assertEquals(RangerMetadataObject.Type.PATH, filesetOwner.get(0).type()); Assertions.assertEquals(3, filesetOwner.get(0).privileges().size()); } From 1fc522d710844e43536989cf2f7899d19b2af73b Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 5 Dec 2024 17:44:56 +0800 Subject: [PATCH 12/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../test/RangerAuthorizationHDFSPluginIT.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index f4bd0bc6423..fbd1dd6a74e 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.gravitino.authorization.ranger.integration.test; import com.google.common.collect.ImmutableList; From f01eb81194f0b86eee7dd41a66a0f02765c62db0 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 5 Dec 2024 19:04:47 +0800 Subject: [PATCH 13/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 35 ++++-- .../RangerAuthorizationHadoopSQLPlugin.java | 66 +++++++---- .../ranger/RangerAuthorizationPlugin.java | 18 +-- .../ranger/RangerHDFSMetadataObject.java | 107 ++++++++++++++++++ .../ranger/RangerHDFSSecurableObject.java | 43 +++++++ ...ava => RangerHadoopSQLMetadataObject.java} | 15 +-- ...va => RangerHadoopSQLSecurableObject.java} | 6 +- .../test/RangerAuthorizationHDFSPluginIT.java | 8 +- .../test/RangerAuthorizationPluginIT.java | 50 ++++---- .../ranger/integration/test/RangerHiveIT.java | 8 +- 10 files changed, 266 insertions(+), 90 deletions(-) create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java create mode 100644 authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSSecurableObject.java rename authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/{RangerMetadataObject.java => RangerHadoopSQLMetadataObject.java} (88%) rename authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/{RangerSecurableObject.java => RangerHadoopSQLSecurableObject.java} (90%) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index e7c48ede3fe..1b08b25c264 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -40,6 +40,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; +import org.apache.gravitino.catalog.FilesetDispatcher; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.file.Fileset; import org.apache.ranger.plugin.model.RangerPolicy; @@ -104,6 +105,18 @@ protected RangerPolicy createPolicyAddResources(AuthorizationMetadataObject meta return policy; } + @Override + public AuthorizationSecurableObject generateAuthorizationSecurableObject( + List names, + AuthorizationMetadataObject.Type type, + Set privileges) { + AuthorizationMetadataObject authMetadataObject = + new RangerHDFSMetadataObject(AuthorizationMetadataObject.getLastName(names), type); + authMetadataObject.validateAuthorizationMetadataObject(); + return new RangerHDFSSecurableObject( + authMetadataObject.name(), authMetadataObject.type(), privileges); + } + @Override public Set allowPrivilegesRule() { return ImmutableSet.of( @@ -151,7 +164,7 @@ public List translatePrivilege(SecurableObject sec rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), - RangerMetadataObject.Type.PATH, + RangerHadoopSQLMetadataObject.Type.PATH, rangerPrivileges)); break; default: @@ -183,7 +196,7 @@ public List translateOwner(MetadataObject gravitin rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), - RangerMetadataObject.Type.PATH, + RangerHadoopSQLMetadataObject.Type.PATH, ownerMappingRule())); break; default: @@ -208,21 +221,23 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada nsMetadataObject.size() > 0, "The metadata object must have at least one name."); nsMetadataObject.remove(0); // Remove the catalog name - RangerMetadataObject rangerMetadataObject = - new RangerMetadataObject( - null, getFileSetPath(metadataObject), RangerMetadataObject.Type.PATH); - rangerMetadataObject.validateAuthorizationMetadataObject(); - return rangerMetadataObject; + RangerHDFSMetadataObject rangerHDFSMetadataObject = + new RangerHDFSMetadataObject( + getFileSetPath(metadataObject), RangerHadoopSQLMetadataObject.Type.PATH); + rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); + return rangerHDFSMetadataObject; } private String getFileSetPath(MetadataObject metadataObject) { - boolean testEnv = System.getenv("GRAVITINO_TEST") != null; - if (testEnv) { + FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); + boolean testEnv = + System.getenv("GRAVITINO_TEST") != null || System.getenv("GRAVITINO_TEST_CLOUD_IT") == null; + if (filesetDispatcher == null && testEnv) { return "/test"; } NameIdentifier identifier = NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); - Fileset fileset = GravitinoEnv.getInstance().filesetDispatcher().loadFileset(identifier); + Fileset fileset = filesetDispatcher.loadFileset(identifier); Preconditions.checkArgument( fileset != null, String.format("Fileset %s is not found", identifier)); String filesetLocation = fileset.storageLocation(); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java index 789fb292f32..0da5c105a4b 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java @@ -118,6 +118,24 @@ protected RangerPolicy createPolicyAddResources(AuthorizationMetadataObject meta return policy; } + @Override + public AuthorizationSecurableObject generateAuthorizationSecurableObject( + List names, + AuthorizationMetadataObject.Type type, + Set privileges) { + AuthorizationMetadataObject authMetadataObject = + new RangerHadoopSQLMetadataObject( + AuthorizationMetadataObject.getParentFullName(names), + AuthorizationMetadataObject.getLastName(names), + type); + authMetadataObject.validateAuthorizationMetadataObject(); + return new RangerHadoopSQLSecurableObject( + authMetadataObject.parent(), + authMetadataObject.name(), + authMetadataObject.type(), + privileges); + } + @Override /** Allow privilege operation defines rule. */ public Set allowPrivilegesRule() { @@ -158,13 +176,13 @@ public List translateOwner(MetadataObject gravitin AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.SCHEMA, + RangerHadoopSQLMetadataObject.Type.SCHEMA, ownerMappingRule())); // Add `*.*` for the TABLE permission AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.TABLE, + RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `*.*.*` for the COLUMN permission AuthorizationSecurableObjects.add( @@ -173,7 +191,7 @@ public List translateOwner(MetadataObject gravitin RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.COLUMN, + RangerHadoopSQLMetadataObject.Type.COLUMN, ownerMappingRule())); break; case SCHEMA: @@ -181,14 +199,14 @@ public List translateOwner(MetadataObject gravitin AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(gravitinoMetadataObject.name() /*Schema name*/), - RangerMetadataObject.Type.SCHEMA, + RangerHadoopSQLMetadataObject.Type.SCHEMA, ownerMappingRule())); // Add `{schema}.*` for the TABLE permission AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( gravitinoMetadataObject.name() /*Schema name*/, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.TABLE, + RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `{schema}.*.*` for the COLUMN permission AuthorizationSecurableObjects.add( @@ -197,7 +215,7 @@ public List translateOwner(MetadataObject gravitin gravitinoMetadataObject.name() /*Schema name*/, RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.COLUMN, + RangerHadoopSQLMetadataObject.Type.COLUMN, ownerMappingRule())); break; case TABLE: @@ -205,7 +223,7 @@ public List translateOwner(MetadataObject gravitin AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), - RangerMetadataObject.Type.TABLE, + RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `{schema}.{table}.*` for the COLUMN permission AuthorizationSecurableObjects.add( @@ -214,7 +232,7 @@ public List translateOwner(MetadataObject gravitin translateMetadataObject(gravitinoMetadataObject).names().stream(), Stream.of(RangerHelper.RESOURCE_ALL)) .collect(Collectors.toList()), - RangerMetadataObject.Type.COLUMN, + RangerHadoopSQLMetadataObject.Type.COLUMN, ownerMappingRule())); break; default: @@ -260,7 +278,7 @@ public List translatePrivilege(SecurableObject sec AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.SCHEMA, + RangerHadoopSQLMetadataObject.Type.SCHEMA, rangerPrivileges)); break; default: @@ -277,7 +295,7 @@ public List translatePrivilege(SecurableObject sec AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.SCHEMA, + RangerHadoopSQLMetadataObject.Type.SCHEMA, rangerPrivileges)); break; default: @@ -294,7 +312,7 @@ public List translatePrivilege(SecurableObject sec AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.SCHEMA, + RangerHadoopSQLMetadataObject.Type.SCHEMA, rangerPrivileges)); break; case SCHEMA: @@ -302,7 +320,7 @@ public List translatePrivilege(SecurableObject sec AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(securableObject.name() /*Schema name*/), - RangerMetadataObject.Type.SCHEMA, + RangerHadoopSQLMetadataObject.Type.SCHEMA, rangerPrivileges)); break; default: @@ -322,7 +340,7 @@ public List translatePrivilege(SecurableObject sec generateAuthorizationSecurableObject( ImmutableList.of( RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.TABLE, + RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `*.*.*` for the COLUMN permission AuthorizationSecurableObjects.add( @@ -331,7 +349,7 @@ public List translatePrivilege(SecurableObject sec RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.COLUMN, + RangerHadoopSQLMetadataObject.Type.COLUMN, rangerPrivileges)); break; case SCHEMA: @@ -341,7 +359,7 @@ public List translatePrivilege(SecurableObject sec ImmutableList.of( securableObject.name() /*Schema name*/, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.TABLE, + RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `{schema}.*.*` for the COLUMN permission AuthorizationSecurableObjects.add( @@ -350,7 +368,7 @@ public List translatePrivilege(SecurableObject sec securableObject.name() /*Schema name*/, RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), - RangerMetadataObject.Type.COLUMN, + RangerHadoopSQLMetadataObject.Type.COLUMN, rangerPrivileges)); break; case TABLE: @@ -363,7 +381,7 @@ public List translatePrivilege(SecurableObject sec AuthorizationSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), - RangerMetadataObject.Type.TABLE, + RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `{schema}.{table}.*` for the COLUMN permission AuthorizationSecurableObjects.add( @@ -372,7 +390,7 @@ public List translatePrivilege(SecurableObject sec translateMetadataObject(securableObject).names().stream(), Stream.of(RangerHelper.RESOURCE_ALL)) .collect(Collectors.toList()), - RangerMetadataObject.Type.COLUMN, + RangerHadoopSQLMetadataObject.Type.COLUMN, rangerPrivileges)); } break; @@ -418,18 +436,18 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada || metadataObject.type() == MetadataObject.Type.CATALOG) { nsMetadataObject.clear(); nsMetadataObject.add(RangerHelper.RESOURCE_ALL); - type = RangerMetadataObject.Type.SCHEMA; + type = RangerHadoopSQLMetadataObject.Type.SCHEMA; } else { nsMetadataObject.remove(0); // Remove the catalog name - type = RangerMetadataObject.Type.fromMetadataType(metadataObject.type()); + type = RangerHadoopSQLMetadataObject.Type.fromMetadataType(metadataObject.type()); } - RangerMetadataObject rangerMetadataObject = - new RangerMetadataObject( + RangerHadoopSQLMetadataObject rangerHadoopSQLMetadataObject = + new RangerHadoopSQLMetadataObject( AuthorizationMetadataObject.getParentFullName(nsMetadataObject), AuthorizationMetadataObject.getLastName(nsMetadataObject), type); - rangerMetadataObject.validateAuthorizationMetadataObject(); - return rangerMetadataObject; + rangerHadoopSQLMetadataObject.validateAuthorizationMetadataObject(); + return rangerHadoopSQLMetadataObject; } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index ef544348fd8..92ee5dd07f3 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -868,7 +868,7 @@ private void doRemoveMetadataObject(AuthorizationMetadataObject authMetadataObje */ private void doRemoveSchemaMetadataObject(AuthorizationMetadataObject authMetadataObject) { Preconditions.checkArgument( - authMetadataObject.type() == RangerMetadataObject.Type.SCHEMA, + authMetadataObject.type() == RangerHadoopSQLMetadataObject.Type.SCHEMA, "The metadata object type must be SCHEMA"); Preconditions.checkArgument( authMetadataObject.names().size() == 1, "The metadata object names must be 1"); @@ -1132,22 +1132,10 @@ private void updatePolicyByMetadataObject( public void close() throws IOException {} /** Generate authorization securable object */ - public AuthorizationSecurableObject generateAuthorizationSecurableObject( + public abstract AuthorizationSecurableObject generateAuthorizationSecurableObject( List names, AuthorizationMetadataObject.Type type, - Set privileges) { - AuthorizationMetadataObject authMetadataObject = - new RangerMetadataObject( - AuthorizationMetadataObject.getParentFullName(names), - AuthorizationMetadataObject.getLastName(names), - type); - authMetadataObject.validateAuthorizationMetadataObject(); - return new RangerSecurableObject( - authMetadataObject.parent(), - authMetadataObject.name(), - authMetadataObject.type(), - privileges); - } + Set privileges); public boolean validAuthorizationOperation(List securableObjects) { return securableObjects.stream() diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java new file mode 100644 index 00000000000..3d7d2311160 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.AuthorizationMetadataObject; + +public class RangerHDFSMetadataObject implements AuthorizationMetadataObject { + /** + * The type of object in the Ranger system. Every type will map one kind of the entity of the + * Gravitino type system. + */ + public enum Type implements AuthorizationMetadataObject.Type { + /** A path is mapped the path of storages like HDFS, S3 etc. */ + PATH(MetadataObject.Type.FILESET); + + private final MetadataObject.Type metadataType; + + Type(MetadataObject.Type type) { + this.metadataType = type; + } + + public MetadataObject.Type metadataObjectType() { + return metadataType; + } + + public static RangerHadoopSQLMetadataObject.Type fromMetadataType( + MetadataObject.Type metadataType) { + for (RangerHadoopSQLMetadataObject.Type type : RangerHadoopSQLMetadataObject.Type.values()) { + if (type.metadataObjectType() == metadataType) { + return type; + } + } + throw new IllegalArgumentException( + "No matching RangerMetadataObject.Type for " + metadataType); + } + } + + private final String path; + + private final AuthorizationMetadataObject.Type type; + + public RangerHDFSMetadataObject(String path, AuthorizationMetadataObject.Type type) { + this.path = path; + this.type = type; + } + + @Nullable + @Override + public String parent() { + return null; + } + + @Override + public String name() { + return this.path; + } + + @Override + public List names() { + return ImmutableList.of(this.path); + } + + @Override + public AuthorizationMetadataObject.Type type() { + return this.type; + } + + @Override + public void validateAuthorizationMetadataObject() throws IllegalArgumentException { + List names = names(); + Preconditions.checkArgument( + names != null && !names.isEmpty(), "Cannot create a Ranger metadata object with no names"); + Preconditions.checkArgument( + names.size() == 3, + "Cannot create a Ranger metadata object with the name length which is 1"); + Preconditions.checkArgument( + type != null, "Cannot create a Ranger metadata object with no type"); + + Preconditions.checkArgument( + type == RangerHadoopSQLMetadataObject.Type.PATH, "it must be the PATH type"); + + for (String name : names) { + Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); + } + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSSecurableObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSSecurableObject.java new file mode 100644 index 00000000000..df1bac73545 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSSecurableObject.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Set; +import org.apache.gravitino.authorization.AuthorizationMetadataObject; +import org.apache.gravitino.authorization.AuthorizationPrivilege; +import org.apache.gravitino.authorization.AuthorizationSecurableObject; + +public class RangerHDFSSecurableObject extends RangerHDFSMetadataObject + implements AuthorizationSecurableObject { + + private final List privileges; + + public RangerHDFSSecurableObject( + String path, AuthorizationMetadataObject.Type type, Set privileges) { + super(path, type); + this.privileges = ImmutableList.copyOf(privileges); + } + + @Override + public List privileges() { + return privileges; + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java similarity index 88% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java rename to authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java index 3378c65e3d1..2621bb55497 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java @@ -24,7 +24,7 @@ import org.apache.gravitino.authorization.AuthorizationMetadataObject; /** The helper class for {@link AuthorizationMetadataObject}. */ -public class RangerMetadataObject implements AuthorizationMetadataObject { +public class RangerHadoopSQLMetadataObject implements AuthorizationMetadataObject { /** * The type of object in the Ranger system. Every type will map one kind of the entity of the * Gravitino type system. @@ -74,7 +74,8 @@ public static Type fromMetadataType(MetadataObject.Type metadataType) { * @param name The name of the metadata object * @param type The type of the metadata object */ - public RangerMetadataObject(String parent, String name, AuthorizationMetadataObject.Type type) { + public RangerHadoopSQLMetadataObject( + String parent, String name, AuthorizationMetadataObject.Type type) { this.parent = parent; this.name = name; this.type = type; @@ -112,15 +113,15 @@ public void validateAuthorizationMetadataObject() throws IllegalArgumentExceptio type != null, "Cannot create a Ranger metadata object with no type"); Preconditions.checkArgument( - names.size() != 1 || type == RangerMetadataObject.Type.SCHEMA || type == Type.PATH, + names.size() != 1 || type == RangerHadoopSQLMetadataObject.Type.SCHEMA || type == Type.PATH, "If the length of names is 1, it must be the SCHEMA type of PATH type"); Preconditions.checkArgument( - names.size() != 2 || type == RangerMetadataObject.Type.TABLE, + names.size() != 2 || type == RangerHadoopSQLMetadataObject.Type.TABLE, "If the length of names is 2, it must be the TABLE type"); Preconditions.checkArgument( - names.size() != 3 || type == RangerMetadataObject.Type.COLUMN, + names.size() != 3 || type == RangerHadoopSQLMetadataObject.Type.COLUMN, "If the length of names is 3, it must be COLUMN"); for (String name : names) { @@ -134,11 +135,11 @@ public boolean equals(Object o) { return true; } - if (!(o instanceof RangerMetadataObject)) { + if (!(o instanceof RangerHadoopSQLMetadataObject)) { return false; } - RangerMetadataObject that = (RangerMetadataObject) o; + RangerHadoopSQLMetadataObject that = (RangerHadoopSQLMetadataObject) o; return java.util.Objects.equals(name, that.name) && java.util.Objects.equals(parent, that.parent) && type == that.type; diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLSecurableObject.java similarity index 90% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObject.java rename to authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLSecurableObject.java index 3a6294f822c..4aabdc4c32d 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLSecurableObject.java @@ -26,8 +26,8 @@ import org.apache.gravitino.authorization.AuthorizationPrivilege; import org.apache.gravitino.authorization.AuthorizationSecurableObject; -/** The helper class for {@link RangerSecurableObject}. */ -public class RangerSecurableObject extends RangerMetadataObject +/** The helper class for {@link RangerHadoopSQLSecurableObject}. */ +public class RangerHadoopSQLSecurableObject extends RangerHadoopSQLMetadataObject implements AuthorizationSecurableObject { private final List privileges; @@ -38,7 +38,7 @@ public class RangerSecurableObject extends RangerMetadataObject * @param name The name of the metadata object * @param type The type of the metadata object */ - public RangerSecurableObject( + public RangerHadoopSQLSecurableObject( String parent, String name, AuthorizationMetadataObject.Type type, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index fbd1dd6a74e..cb477ac38ed 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -29,7 +29,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; -import org.apache.gravitino.authorization.ranger.RangerMetadataObject; +import org.apache.gravitino.authorization.ranger.RangerHadoopSQLMetadataObject; import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -75,7 +75,7 @@ public void testTranslateMetadataObject() { AuthorizationMetadataObject rangerFileset = rangerAuthPlugin.translateMetadataObject(fileset); Assertions.assertEquals(1, rangerFileset.names().size()); Assertions.assertEquals("/test", rangerFileset.fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.PATH, rangerFileset.type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.PATH, rangerFileset.type()); } @Test @@ -130,7 +130,7 @@ public void testTranslatePrivilege() { filesetInFileset1.forEach( securableObject -> { - Assertions.assertEquals(RangerMetadataObject.Type.PATH, securableObject.type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.PATH, securableObject.type()); Assertions.assertEquals("/test", securableObject.fullName()); Assertions.assertTrue( securableObject.privileges().size() == 1 || securableObject.privileges().size() == 2); @@ -176,7 +176,7 @@ public void testTranslateOwner() { List filesetOwner = rangerAuthPlugin.translateOwner(fileset); Assertions.assertEquals(1, filesetOwner.size()); Assertions.assertEquals("/test", filesetOwner.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.PATH, filesetOwner.get(0).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.PATH, filesetOwner.get(0).type()); Assertions.assertEquals(3, filesetOwner.get(0).privileges().size()); } } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java index 50ca331d221..e7a4f03bc3d 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java @@ -31,8 +31,8 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; +import org.apache.gravitino.authorization.ranger.RangerHadoopSQLMetadataObject; import org.apache.gravitino.authorization.ranger.RangerHelper; -import org.apache.gravitino.authorization.ranger.RangerMetadataObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -55,21 +55,21 @@ public void testTranslateMetadataObject() { AuthorizationMetadataObject rangerMetalake = rangerAuthPlugin.translateMetadataObject(metalake); Assertions.assertEquals(1, rangerMetalake.names().size()); Assertions.assertEquals(RangerHelper.RESOURCE_ALL, rangerMetalake.names().get(0)); - Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, rangerMetalake.type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.SCHEMA, rangerMetalake.type()); MetadataObject catalog = MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); AuthorizationMetadataObject rangerCatalog = rangerAuthPlugin.translateMetadataObject(catalog); Assertions.assertEquals(1, rangerCatalog.names().size()); Assertions.assertEquals(RangerHelper.RESOURCE_ALL, rangerCatalog.names().get(0)); - Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, rangerCatalog.type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.SCHEMA, rangerCatalog.type()); MetadataObject schema = MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); AuthorizationMetadataObject rangerSchema = rangerAuthPlugin.translateMetadataObject(schema); Assertions.assertEquals(1, rangerSchema.names().size()); Assertions.assertEquals("schema1", rangerSchema.names().get(0)); - Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, rangerSchema.type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.SCHEMA, rangerSchema.type()); MetadataObject table = MetadataObjects.parse(String.format("catalog1.schema1.tab1"), MetadataObject.Type.TABLE); @@ -77,7 +77,7 @@ public void testTranslateMetadataObject() { Assertions.assertEquals(2, rangerTable.names().size()); Assertions.assertEquals("schema1", rangerTable.names().get(0)); Assertions.assertEquals("tab1", rangerTable.names().get(1)); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, rangerTable.type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.TABLE, rangerTable.type()); } @Test @@ -92,7 +92,7 @@ public void testTranslatePrivilege() { Assertions.assertEquals(1, createSchemaInMetalake1.size()); Assertions.assertEquals(RangerHelper.RESOURCE_ALL, createSchemaInMetalake1.get(0).fullName()); Assertions.assertEquals( - RangerMetadataObject.Type.SCHEMA, createSchemaInMetalake1.get(0).type()); + RangerHadoopSQLMetadataObject.Type.SCHEMA, createSchemaInMetalake1.get(0).type()); SecurableObject createSchemaInCatalog = SecurableObjects.parse( @@ -103,7 +103,8 @@ public void testTranslatePrivilege() { rangerAuthPlugin.translatePrivilege(createSchemaInCatalog); Assertions.assertEquals(1, createSchemaInCatalog1.size()); Assertions.assertEquals(RangerHelper.RESOURCE_ALL, createSchemaInCatalog1.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, createSchemaInCatalog1.get(0).type()); + Assertions.assertEquals( + RangerHadoopSQLMetadataObject.Type.SCHEMA, createSchemaInCatalog1.get(0).type()); for (Privilege privilege : ImmutableList.of( @@ -118,9 +119,9 @@ public void testTranslatePrivilege() { List metalake1 = rangerAuthPlugin.translatePrivilege(metalake); Assertions.assertEquals(2, metalake1.size()); Assertions.assertEquals("*.*", metalake1.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, metalake1.get(0).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.TABLE, metalake1.get(0).type()); Assertions.assertEquals("*.*.*", metalake1.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.COLUMN, metalake1.get(1).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.COLUMN, metalake1.get(1).type()); SecurableObject catalog = SecurableObjects.parse( @@ -130,9 +131,9 @@ public void testTranslatePrivilege() { List catalog1 = rangerAuthPlugin.translatePrivilege(catalog); Assertions.assertEquals(2, catalog1.size()); Assertions.assertEquals("*.*", catalog1.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, catalog1.get(0).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.TABLE, catalog1.get(0).type()); Assertions.assertEquals("*.*.*", catalog1.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.COLUMN, catalog1.get(1).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.COLUMN, catalog1.get(1).type()); SecurableObject schema = SecurableObjects.parse( @@ -142,9 +143,9 @@ public void testTranslatePrivilege() { List schema1 = rangerAuthPlugin.translatePrivilege(schema); Assertions.assertEquals(2, schema1.size()); Assertions.assertEquals("schema1.*", schema1.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, schema1.get(0).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.TABLE, schema1.get(0).type()); Assertions.assertEquals("schema1.*.*", schema1.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.COLUMN, schema1.get(1).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.COLUMN, schema1.get(1).type()); if (!privilege.equals(Privileges.CreateTable.allow())) { // `CREATE_TABLE` not support securable object for table, So ignore check for table. @@ -156,9 +157,9 @@ public void testTranslatePrivilege() { List table1 = rangerAuthPlugin.translatePrivilege(table); Assertions.assertEquals(2, table1.size()); Assertions.assertEquals("schema1.table1", table1.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, table1.get(0).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.TABLE, table1.get(0).type()); Assertions.assertEquals("schema1.table1.*", table1.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.COLUMN, table1.get(1).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.COLUMN, table1.get(1).type()); } } } @@ -171,31 +172,34 @@ public void testTranslateOwner() { List metalakeOwner = rangerAuthPlugin.translateOwner(metalake); Assertions.assertEquals(3, metalakeOwner.size()); Assertions.assertEquals(RangerHelper.RESOURCE_ALL, metalakeOwner.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, metalakeOwner.get(0).type()); + Assertions.assertEquals( + RangerHadoopSQLMetadataObject.Type.SCHEMA, metalakeOwner.get(0).type()); Assertions.assertEquals("*.*", metalakeOwner.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, metalakeOwner.get(1).type()); + Assertions.assertEquals( + RangerHadoopSQLMetadataObject.Type.TABLE, metalakeOwner.get(1).type()); Assertions.assertEquals("*.*.*", metalakeOwner.get(2).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.COLUMN, metalakeOwner.get(2).type()); + Assertions.assertEquals( + RangerHadoopSQLMetadataObject.Type.COLUMN, metalakeOwner.get(2).type()); } MetadataObject schema = MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA); List schemaOwner = rangerAuthPlugin.translateOwner(schema); Assertions.assertEquals(3, schemaOwner.size()); Assertions.assertEquals("schema1", schemaOwner.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, schemaOwner.get(0).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.SCHEMA, schemaOwner.get(0).type()); Assertions.assertEquals("schema1.*", schemaOwner.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, schemaOwner.get(1).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.TABLE, schemaOwner.get(1).type()); Assertions.assertEquals("schema1.*.*", schemaOwner.get(2).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.COLUMN, schemaOwner.get(2).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.COLUMN, schemaOwner.get(2).type()); MetadataObject table = MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE); List tableOwner = rangerAuthPlugin.translateOwner(table); Assertions.assertEquals(2, tableOwner.size()); Assertions.assertEquals("schema1.table1", tableOwner.get(0).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.TABLE, tableOwner.get(0).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.TABLE, tableOwner.get(0).type()); Assertions.assertEquals("schema1.table1.*", tableOwner.get(1).fullName()); - Assertions.assertEquals(RangerMetadataObject.Type.COLUMN, tableOwner.get(1).type()); + Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.COLUMN, tableOwner.get(1).type()); } @Test diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index dce93a6142d..356e34d2824 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -48,10 +48,10 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; +import org.apache.gravitino.authorization.ranger.RangerHadoopSQLMetadataObject; +import org.apache.gravitino.authorization.ranger.RangerHadoopSQLSecurableObject; import org.apache.gravitino.authorization.ranger.RangerHelper; -import org.apache.gravitino.authorization.ranger.RangerMetadataObject; import org.apache.gravitino.authorization.ranger.RangerPrivileges; -import org.apache.gravitino.authorization.ranger.RangerSecurableObject; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.meta.AuditInfo; @@ -343,7 +343,7 @@ public void testFindManagedPolicy() { AuthorizationSecurableObject rangerSecurableObject = rangerAuthHivePlugin.generateAuthorizationSecurableObject( ImmutableList.of(String.format("%s3", dbName), "tab1"), - RangerMetadataObject.Type.TABLE, + RangerHadoopSQLMetadataObject.Type.TABLE, ImmutableSet.of( new RangerPrivileges.RangerHivePrivilegeImpl( RangerPrivileges.RangerHadoopSQLPrivilege.ALL, Privilege.Condition.ALLOW))); @@ -460,7 +460,7 @@ static void createHivePolicy(List metaObjects, String roleName) { Collections.singletonList(policyItem)); } - static boolean deleteHivePolicy(RangerSecurableObject rangerSecurableObject) { + static boolean deleteHivePolicy(RangerHadoopSQLSecurableObject rangerSecurableObject) { RangerPolicy policy = rangerHelper.findManagedPolicy(rangerSecurableObject); if (policy != null) { try { From 6e3ab7a30f03546c61dc7238487f60629f392027 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 5 Dec 2024 19:13:15 +0800 Subject: [PATCH 14/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../authorization/ranger/RangerHDFSMetadataObject.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java index 3d7d2311160..9059e3f31f5 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java @@ -92,7 +92,7 @@ public void validateAuthorizationMetadataObject() throws IllegalArgumentExceptio Preconditions.checkArgument( names != null && !names.isEmpty(), "Cannot create a Ranger metadata object with no names"); Preconditions.checkArgument( - names.size() == 3, + names.size() == 1, "Cannot create a Ranger metadata object with the name length which is 1"); Preconditions.checkArgument( type != null, "Cannot create a Ranger metadata object with no type"); From 07bb69d9259b3538c814fcec42d081430088c43b Mon Sep 17 00:00:00 2001 From: theoryxu Date: Fri, 6 Dec 2024 10:03:34 +0800 Subject: [PATCH 15/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 6 +++--- .../authorization/ranger/RangerHDFSMetadataObject.java | 2 +- .../ranger/RangerHadoopSQLMetadataObject.java | 9 +++------ .../test/RangerAuthorizationHDFSPluginIT.java | 8 ++++---- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 1b08b25c264..ea678aebdbf 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -164,7 +164,7 @@ public List translatePrivilege(SecurableObject sec rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), - RangerHadoopSQLMetadataObject.Type.PATH, + RangerHDFSMetadataObject.Type.PATH, rangerPrivileges)); break; default: @@ -196,7 +196,7 @@ public List translateOwner(MetadataObject gravitin rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), - RangerHadoopSQLMetadataObject.Type.PATH, + RangerHDFSMetadataObject.Type.PATH, ownerMappingRule())); break; default: @@ -223,7 +223,7 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada nsMetadataObject.remove(0); // Remove the catalog name RangerHDFSMetadataObject rangerHDFSMetadataObject = new RangerHDFSMetadataObject( - getFileSetPath(metadataObject), RangerHadoopSQLMetadataObject.Type.PATH); + getFileSetPath(metadataObject), RangerHDFSMetadataObject.Type.PATH); rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); return rangerHDFSMetadataObject; } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java index 9059e3f31f5..aa9e1b1233c 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java @@ -98,7 +98,7 @@ public void validateAuthorizationMetadataObject() throws IllegalArgumentExceptio type != null, "Cannot create a Ranger metadata object with no type"); Preconditions.checkArgument( - type == RangerHadoopSQLMetadataObject.Type.PATH, "it must be the PATH type"); + type == RangerHDFSMetadataObject.Type.PATH, "it must be the PATH type"); for (String name : names) { Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java index 2621bb55497..8462a0e07a5 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java @@ -35,10 +35,7 @@ public enum Type implements AuthorizationMetadataObject.Type { /** A table is mapped the table of relational data sources like Apache Hive, MySQL, etc. */ TABLE(MetadataObject.Type.TABLE), /** A column is a sub-collection of the table that represents a group of same type data. */ - COLUMN(MetadataObject.Type.COLUMN), - /** A path is mapped the path of storages like HDFS, S3 etc. */ - PATH(MetadataObject.Type.FILESET); - + COLUMN(MetadataObject.Type.COLUMN); private final MetadataObject.Type metadataType; Type(MetadataObject.Type type) { @@ -113,8 +110,8 @@ public void validateAuthorizationMetadataObject() throws IllegalArgumentExceptio type != null, "Cannot create a Ranger metadata object with no type"); Preconditions.checkArgument( - names.size() != 1 || type == RangerHadoopSQLMetadataObject.Type.SCHEMA || type == Type.PATH, - "If the length of names is 1, it must be the SCHEMA type of PATH type"); + names.size() != 1 || type == RangerHadoopSQLMetadataObject.Type.SCHEMA, + "If the length of names is 1, it must be the SCHEMA type"); Preconditions.checkArgument( names.size() != 2 || type == RangerHadoopSQLMetadataObject.Type.TABLE, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index cb477ac38ed..ff40eb6c8ca 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -29,7 +29,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; -import org.apache.gravitino.authorization.ranger.RangerHadoopSQLMetadataObject; +import org.apache.gravitino.authorization.ranger.RangerHDFSMetadataObject; import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -75,7 +75,7 @@ public void testTranslateMetadataObject() { AuthorizationMetadataObject rangerFileset = rangerAuthPlugin.translateMetadataObject(fileset); Assertions.assertEquals(1, rangerFileset.names().size()); Assertions.assertEquals("/test", rangerFileset.fullName()); - Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.PATH, rangerFileset.type()); + Assertions.assertEquals(RangerHDFSMetadataObject.Type.PATH, rangerFileset.type()); } @Test @@ -130,7 +130,7 @@ public void testTranslatePrivilege() { filesetInFileset1.forEach( securableObject -> { - Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.PATH, securableObject.type()); + Assertions.assertEquals(RangerHDFSMetadataObject.Type.PATH, securableObject.type()); Assertions.assertEquals("/test", securableObject.fullName()); Assertions.assertTrue( securableObject.privileges().size() == 1 || securableObject.privileges().size() == 2); @@ -176,7 +176,7 @@ public void testTranslateOwner() { List filesetOwner = rangerAuthPlugin.translateOwner(fileset); Assertions.assertEquals(1, filesetOwner.size()); Assertions.assertEquals("/test", filesetOwner.get(0).fullName()); - Assertions.assertEquals(RangerHadoopSQLMetadataObject.Type.PATH, filesetOwner.get(0).type()); + Assertions.assertEquals(RangerHDFSMetadataObject.Type.PATH, filesetOwner.get(0).type()); Assertions.assertEquals(3, filesetOwner.get(0).privileges().size()); } } From 4e711af6b63adbba50cd9f1aa8b7df27c14c95be Mon Sep 17 00:00:00 2001 From: theoryxu Date: Fri, 6 Dec 2024 15:13:56 +0800 Subject: [PATCH 16/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../integration/test/RangerFilesetE2EIT.java | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java new file mode 100644 index 00000000000..b8bfa77f313 --- /dev/null +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java @@ -0,0 +1,221 @@ +package org.apache.gravitino.authorization.ranger.integration.test; + +import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; +import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; +import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; +import static org.apache.gravitino.connector.AuthorizationPropertiesMeta.RANGER_AUTH_TYPE; +import static org.apache.gravitino.connector.AuthorizationPropertiesMeta.RANGER_PASSWORD; +import static org.apache.gravitino.connector.AuthorizationPropertiesMeta.RANGER_SERVICE_NAME; +import static org.apache.gravitino.connector.AuthorizationPropertiesMeta.RANGER_USERNAME; +import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.Configs; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Schema; +import org.apache.gravitino.auth.AuthConstants; +import org.apache.gravitino.auth.AuthenticatorType; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.connector.AuthorizationPropertiesMeta; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag("gravitino-docker-test") +public class RangerFilesetE2EIT extends BaseIT { + private static final Logger LOG = LoggerFactory.getLogger(RangerFilesetE2EIT.class); + + private String RANGER_ADMIN_URL; + private String HADOOP_USER_NAME = "HADOOP_USER_NAME"; + private String defaultBaseLocation; + private String metalakeName = GravitinoITUtils.genRandomName("metalake").toLowerCase(); + private String catalogName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_catalog"); + private String schemaName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_schema"); + private static final String provider = "hadoop"; + private FileSystem fileSystem; + private GravitinoMetalake metalake; + private Catalog catalog; + + @BeforeAll + public void startIntegrationTest() throws Exception { + // Enable Gravitino Authorization mode + Map configs = Maps.newHashMap(); + configs.put(Configs.ENABLE_AUTHORIZATION.getKey(), String.valueOf(true)); + configs.put(Configs.SERVICE_ADMINS.getKey(), RangerITEnv.HADOOP_USER_NAME); + configs.put(Configs.AUTHENTICATORS.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase()); + configs.put("SimpleAuthUserName", AuthConstants.ANONYMOUS_USER); + registerCustomConfigs(configs); + super.startIntegrationTest(); + + RangerITEnv.init(); + RangerITEnv.startHiveRangerContainer(); + + RANGER_ADMIN_URL = + String.format( + "http://%s:%d", + containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); + + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", defaultBaseLocation()); + fileSystem = FileSystem.get(conf); + + createCatalogAndSchema(); + } + + @AfterAll + public void stop() throws IOException { + if (client != null) { + Arrays.stream(catalog.asSchemas().listSchemas()) + .filter(schema -> !schema.equals("default")) + .forEach( + (schema -> { + catalog.asSchemas().dropSchema(schema, false); + })); + Arrays.stream(metalake.listCatalogs()) + .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); + client.disableMetalake(metalakeName); + client.dropMetalake(metalakeName); + } + if (fileSystem != null) { + fileSystem.close(); + } + try { + closer.close(); + } catch (Exception e) { + LOG.error("Failed to close CloseableGroup", e); + } + client = null; + RangerITEnv.cleanup(); + } + + @Test + void testReadWritePath() throws InterruptedException, IOException { + String filename = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_fileset"); + Fileset fileset = createFileset(filename, Fileset.Type.MANAGED, storageLocation(filename)); + Assertions.assertTrue(fileSystem.exists(new Path(storageLocation(filename)))); + + Assertions.assertThrows( + Exception.class, () -> fileSystem.listFiles(new Path(storageLocation(filename)), true)); + Assertions.assertThrows( + Exception.class, () -> fileSystem.mkdirs(new Path(storageLocation(filename) + "/test"))); + + String filesetRole = currentFunName(); + SecurableObject securableObject = + SecurableObjects.parse( + fileset.name(), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.ReadFileset.allow())); + String userName1 = System.getenv(HADOOP_USER_NAME); + metalake.createRole(filesetRole, Collections.emptyMap(), Lists.newArrayList(securableObject)); + metalake.grantRolesToUser(Lists.newArrayList(filesetRole), userName1); + waitForUpdatingPolicies(); + Assertions.assertNotNull(fileSystem.listFiles(new Path(storageLocation(filename)), true)); + Assertions.assertThrows( + Exception.class, () -> fileSystem.mkdirs(new Path(storageLocation(filename) + "/test"))); + + metalake.grantPrivilegesToRole( + filesetRole, + MetadataObjects.of( + String.format("%s.%s", catalogName, schemaName), + fileset.name(), + MetadataObject.Type.FILESET), + Lists.newArrayList(Privileges.WriteFileset.allow())); + waitForUpdatingPolicies(); + Assertions.assertNotNull(fileSystem.listFiles(new Path(storageLocation(filename)), true)); + Assertions.assertTrue(fileSystem.mkdirs(new Path(storageLocation(filename) + "/test"))); + } + + private void createCatalogAndSchema() { + GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); + Assertions.assertEquals(0, gravitinoMetalakes.length); + + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); + Assertions.assertEquals(metalakeName, loadMetalake.name()); + + metalake = loadMetalake; + metalake.createCatalog( + catalogName, + Catalog.Type.FILESET, + provider, + "comment", + ImmutableMap.of( + IMPERSONATION_ENABLE, + "true", + AUTHORIZATION_PROVIDER, + "ranger", + RANGER_SERVICE_NAME, + RangerITEnv.RANGER_HDFS_REPO_NAME, + AuthorizationPropertiesMeta.RANGER_ADMIN_URL, + RANGER_ADMIN_URL, + RANGER_AUTH_TYPE, + RangerContainer.authType, + RANGER_USERNAME, + RangerContainer.rangerUserName, + RANGER_PASSWORD, + RangerContainer.rangerPassword)); + + catalog = metalake.loadCatalog(catalogName); + catalog + .asSchemas() + .createSchema(schemaName, "comment", ImmutableMap.of("location", defaultBaseLocation())); + Schema loadSchema = catalog.asSchemas().loadSchema(schemaName); + Assertions.assertEquals(schemaName, loadSchema.name()); + Assertions.assertNotNull(loadSchema.properties().get("location")); + } + + private String defaultBaseLocation() { + if (defaultBaseLocation == null) { + defaultBaseLocation = + String.format( + "hdfs://%s:%d/user/hadoop/%s", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT, + schemaName.toLowerCase()); + } + return defaultBaseLocation; + } + + private Fileset createFileset(String filesetName, Fileset.Type type, String storageLocation) { + return catalog + .asFilesetCatalog() + .createFileset( + NameIdentifier.of(schemaName, filesetName), "comment", type, storageLocation, null); + } + + private String storageLocation(String filesetName) { + return defaultBaseLocation() + "/" + filesetName; + } + + private void waitForUpdatingPolicies() throws InterruptedException { + // After Ranger authorization, Must wait a period of time for the Ranger Spark plugin to update + // the policy Sleep time must be greater than the policy update interval + // (ranger.plugin.spark.policy.pollIntervalMs) in the + // `resources/ranger-spark-security.xml.template` + Thread.sleep(1000L); + } +} From 10617a64e797279b73911930f2e285929f6f7c7a Mon Sep 17 00:00:00 2001 From: theoryxu Date: Fri, 6 Dec 2024 15:16:13 +0800 Subject: [PATCH 17/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../integration/test/RangerFilesetE2EIT.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java index b8bfa77f313..475bcdeb981 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.gravitino.authorization.ranger.integration.test; import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; From 9e987f9af95efde36d6b12a05a90b672a2c9afba Mon Sep 17 00:00:00 2001 From: theoryxu Date: Mon, 9 Dec 2024 07:31:49 +0800 Subject: [PATCH 18/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 21 +- .../ranger/RangerAuthorizationPlugin.java | 52 +++-- .../ranger/RangerHDFSMetadataObject.java | 4 +- .../test/RangerAuthorizationHDFSPluginIT.java | 2 +- .../test/RangerAuthorizationPluginIT.java | 2 +- ...FilesetE2EIT.java => RangerFilesetIT.java} | 201 +++++++++++++++--- .../integration/test/RangerHiveE2EIT.java | 2 +- .../ranger/integration/test/RangerHiveIT.java | 2 +- .../ranger/integration/test/RangerITEnv.java | 6 +- .../integration/test/RangerIcebergE2EIT.java | 2 +- .../integration/test/RangerPaimonE2EIT.java | 2 +- 11 files changed, 238 insertions(+), 58 deletions(-) rename authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/{RangerFilesetE2EIT.java => RangerFilesetIT.java} (52%) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index ea678aebdbf..4273723eece 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -125,7 +125,11 @@ public Set allowPrivilegesRule() { @Override public Set allowMetadataObjectTypesRule() { - return ImmutableSet.of(MetadataObject.Type.FILESET); + return ImmutableSet.of( + MetadataObject.Type.FILESET, + MetadataObject.Type.SCHEMA, + MetadataObject.Type.CATALOG, + MetadataObject.Type.METALAKE); } @Override @@ -220,12 +224,15 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada Preconditions.checkArgument( nsMetadataObject.size() > 0, "The metadata object must have at least one name."); - nsMetadataObject.remove(0); // Remove the catalog name - RangerHDFSMetadataObject rangerHDFSMetadataObject = - new RangerHDFSMetadataObject( - getFileSetPath(metadataObject), RangerHDFSMetadataObject.Type.PATH); - rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); - return rangerHDFSMetadataObject; + if (metadataObject.type() == MetadataObject.Type.FILESET) { + RangerHDFSMetadataObject rangerHDFSMetadataObject = + new RangerHDFSMetadataObject( + getFileSetPath(metadataObject), RangerHDFSMetadataObject.Type.PATH); + rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); + return rangerHDFSMetadataObject; + } else { + return new RangerHDFSMetadataObject("", RangerHDFSMetadataObject.Type.SCHEMA_PATH); + } } private String getFileSetPath(MetadataObject metadataObject) { diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 92ee5dd07f3..baaa4beff65 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -328,9 +328,11 @@ public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws Runtime } else if (change instanceof MetadataObjectChange.RemoveMetadataObject) { MetadataObject metadataObject = ((MetadataObjectChange.RemoveMetadataObject) change).metadataObject(); - AuthorizationMetadataObject AuthorizationMetadataObject = - translateMetadataObject(metadataObject); - doRemoveMetadataObject(AuthorizationMetadataObject); + if (metadataObject.type() != MetadataObject.Type.FILESET) { + AuthorizationMetadataObject AuthorizationMetadataObject = + translateMetadataObject(metadataObject); + doRemoveMetadataObject(AuthorizationMetadataObject); + } } else { throw new IllegalArgumentException( "Unsupported metadata object change type: " @@ -846,19 +848,34 @@ private void removePolicyItemIfEqualRoleName( * IF remove the COLUMN, Only need to remove `{schema}.*.*`
*/ private void doRemoveMetadataObject(AuthorizationMetadataObject authMetadataObject) { - switch (authMetadataObject.metadataObjectType()) { - case SCHEMA: - doRemoveSchemaMetadataObject(authMetadataObject); - break; - case TABLE: - doRemoveTableMetadataObject(authMetadataObject); - break; - case COLUMN: - removePolicyByMetadataObject(authMetadataObject.names()); - break; - default: - throw new IllegalArgumentException( - "Unsupported metadata object type: " + authMetadataObject.type()); + if (authMetadataObject instanceof RangerHadoopSQLMetadataObject) { + switch (authMetadataObject.metadataObjectType()) { + case SCHEMA: + doRemoveSchemaMetadataObject(authMetadataObject); + break; + case TABLE: + doRemoveTableMetadataObject(authMetadataObject); + break; + case COLUMN: + removePolicyByMetadataObject(authMetadataObject.names()); + break; + default: + throw new IllegalArgumentException( + "Unsupported metadata object type: " + authMetadataObject.type()); + } + } else if (authMetadataObject instanceof RangerHDFSMetadataObject) { + switch (authMetadataObject.metadataObjectType()) { + case FILESET: + removePolicyByMetadataObject(authMetadataObject.names()); + break; + default: + LOG.info( + "type {} do nothing in RangerHDFSMetadataObject", + authMetadataObject.metadataObjectType()); + } + } else { + throw new IllegalArgumentException( + "Unsupported authorization Metadata object: " + authMetadataObject); } } @@ -943,6 +960,9 @@ private void doRenameMetadataObject( case COLUMN: doRenameColumnMetadataObject(AuthorizationMetadataObject, newAuthMetadataObject); break; + case FILESET: + // do nothing when fileset is renamed + break; default: throw new IllegalArgumentException( "Unsupported metadata object type: " + AuthorizationMetadataObject.type()); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java index aa9e1b1233c..419382083b4 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java @@ -32,8 +32,8 @@ public class RangerHDFSMetadataObject implements AuthorizationMetadataObject { */ public enum Type implements AuthorizationMetadataObject.Type { /** A path is mapped the path of storages like HDFS, S3 etc. */ - PATH(MetadataObject.Type.FILESET); - + PATH(MetadataObject.Type.FILESET), + SCHEMA_PATH(MetadataObject.Type.SCHEMA); private final MetadataObject.Type metadataType; Type(MetadataObject.Type type) { diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index ff40eb6c8ca..9ffbf89d433 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -43,7 +43,7 @@ public class RangerAuthorizationHDFSPluginIT { @BeforeAll public static void setup() { - RangerITEnv.init(); + RangerITEnv.init(true); rangerAuthPlugin = RangerITEnv.rangerAuthHDFSPlugin; } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java index e7a4f03bc3d..f3a474f213d 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java @@ -44,7 +44,7 @@ public class RangerAuthorizationPluginIT { @BeforeAll public static void setup() { - RangerITEnv.init(); + RangerITEnv.init(true); rangerAuthPlugin = RangerITEnv.rangerAuthHivePlugin; } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java similarity index 52% rename from authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java rename to authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index 475bcdeb981..b9aa93a707a 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -20,6 +20,8 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; +import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.rangerClient; +import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.rangerHelper; import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; import static org.apache.gravitino.connector.AuthorizationPropertiesMeta.RANGER_AUTH_TYPE; import static org.apache.gravitino.connector.AuthorizationPropertiesMeta.RANGER_PASSWORD; @@ -33,6 +35,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.Configs; @@ -45,6 +48,7 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.ranger.RangerHelper; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.connector.AuthorizationPropertiesMeta; import org.apache.gravitino.file.Fileset; @@ -55,6 +59,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.ranger.RangerServiceException; +import org.apache.ranger.plugin.model.RangerPolicy; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -64,13 +70,12 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") -public class RangerFilesetE2EIT extends BaseIT { - private static final Logger LOG = LoggerFactory.getLogger(RangerFilesetE2EIT.class); +public class RangerFilesetIT extends BaseIT { + private static final Logger LOG = LoggerFactory.getLogger(RangerFilesetIT.class); private String RANGER_ADMIN_URL; - private String HADOOP_USER_NAME = "HADOOP_USER_NAME"; private String defaultBaseLocation; - private String metalakeName = GravitinoITUtils.genRandomName("metalake").toLowerCase(); + private String metalakeName = "metalake"; private String catalogName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_catalog"); private String schemaName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_schema"); private static final String provider = "hadoop"; @@ -89,7 +94,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(); + RangerITEnv.init(false); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = @@ -131,29 +136,95 @@ public void stop() throws IOException { } @Test - void testReadWritePath() throws InterruptedException, IOException { + void testReadWritePath() throws IOException, RangerServiceException { String filename = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_fileset"); - Fileset fileset = createFileset(filename, Fileset.Type.MANAGED, storageLocation(filename)); + Fileset fileset = + catalog + .asFilesetCatalog() + .createFileset( + NameIdentifier.of(schemaName, filename), + "comment", + Fileset.Type.MANAGED, + storageLocation(filename), + null); + Assertions.assertTrue( + catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, fileset.name()))); Assertions.assertTrue(fileSystem.exists(new Path(storageLocation(filename)))); + List policies = + rangerClient.getPoliciesInService(RangerITEnv.RANGER_HDFS_REPO_NAME); + Assertions.assertEquals(1, policies.size()); + Assertions.assertEquals(3, policies.get(0).getPolicyItems().size()); - Assertions.assertThrows( - Exception.class, () -> fileSystem.listFiles(new Path(storageLocation(filename)), true)); - Assertions.assertThrows( - Exception.class, () -> fileSystem.mkdirs(new Path(storageLocation(filename) + "/test"))); + Assertions.assertEquals( + 1, + policies.get(0).getPolicyItems().stream() + .filter(item -> item.getRoles().contains(RangerHelper.GRAVITINO_OWNER_ROLE)) + .filter( + item -> + item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + .count()); + Assertions.assertEquals( + 1, + policies.get(0).getPolicyItems().stream() + .filter(item -> item.getRoles().contains(RangerHelper.GRAVITINO_OWNER_ROLE)) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("write"))) + .count()); + Assertions.assertEquals( + 1, + policies.get(0).getPolicyItems().stream() + .filter(item -> item.getRoles().contains(RangerHelper.GRAVITINO_OWNER_ROLE)) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("execute"))) + .count()); String filesetRole = currentFunName(); SecurableObject securableObject = SecurableObjects.parse( - fileset.name(), + String.format("%s.%s.%s", catalogName, schemaName, fileset.name()), MetadataObject.Type.FILESET, Lists.newArrayList(Privileges.ReadFileset.allow())); - String userName1 = System.getenv(HADOOP_USER_NAME); metalake.createRole(filesetRole, Collections.emptyMap(), Lists.newArrayList(securableObject)); - metalake.grantRolesToUser(Lists.newArrayList(filesetRole), userName1); - waitForUpdatingPolicies(); - Assertions.assertNotNull(fileSystem.listFiles(new Path(storageLocation(filename)), true)); - Assertions.assertThrows( - Exception.class, () -> fileSystem.mkdirs(new Path(storageLocation(filename) + "/test"))); + + policies = rangerClient.getPoliciesInService(RangerITEnv.RANGER_HDFS_REPO_NAME); + Assertions.assertEquals(1, policies.size()); + Assertions.assertEquals(3, policies.get(0).getPolicyItems().size()); + Assertions.assertEquals( + 1, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + .count()); + Assertions.assertEquals( + 0, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("write"))) + .count()); + Assertions.assertEquals( + 0, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("execute"))) + .count()); metalake.grantPrivilegesToRole( filesetRole, @@ -162,9 +233,90 @@ void testReadWritePath() throws InterruptedException, IOException { fileset.name(), MetadataObject.Type.FILESET), Lists.newArrayList(Privileges.WriteFileset.allow())); - waitForUpdatingPolicies(); - Assertions.assertNotNull(fileSystem.listFiles(new Path(storageLocation(filename)), true)); - Assertions.assertTrue(fileSystem.mkdirs(new Path(storageLocation(filename) + "/test"))); + + policies = rangerClient.getPoliciesInService(RangerITEnv.RANGER_HDFS_REPO_NAME); + Assertions.assertEquals(1, policies.size()); + Assertions.assertEquals(3, policies.get(0).getPolicyItems().size()); + Assertions.assertEquals( + 1, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + .count()); + Assertions.assertEquals( + 1, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("write"))) + .count()); + Assertions.assertEquals( + 1, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("execute"))) + .count()); + + metalake.revokePrivilegesFromRole( + filesetRole, + MetadataObjects.of( + String.format("%s.%s", catalogName, schemaName), + fileset.name(), + MetadataObject.Type.FILESET), + Lists.newArrayList(Privileges.ReadFileset.allow(), Privileges.WriteFileset.allow())); + policies = rangerClient.getPoliciesInService(RangerITEnv.RANGER_HDFS_REPO_NAME); + Assertions.assertEquals(1, policies.size()); + Assertions.assertEquals(3, policies.get(0).getPolicyItems().size()); + Assertions.assertEquals( + 0, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + .count()); + Assertions.assertEquals( + 0, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("write"))) + .count()); + Assertions.assertEquals( + 0, + policies.get(0).getPolicyItems().stream() + .filter( + item -> + item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) + .filter( + item -> + item.getAccesses().stream() + .anyMatch(access -> access.getType().equals("execute"))) + .count()); + + catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(schemaName, fileset.name())); + policies = rangerClient.getPoliciesInService(RangerITEnv.RANGER_HDFS_REPO_NAME); + Assertions.assertEquals(1, policies.size()); + Assertions.assertEquals(3, policies.get(0).getPolicyItems().size()); } private void createCatalogAndSchema() { @@ -172,10 +324,9 @@ private void createCatalogAndSchema() { Assertions.assertEquals(0, gravitinoMetalakes.length); client.createMetalake(metalakeName, "comment", Collections.emptyMap()); - GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(metalakeName, loadMetalake.name()); + metalake = client.loadMetalake(metalakeName); + Assertions.assertEquals(metalakeName, metalake.name()); - metalake = loadMetalake; metalake.createCatalog( catalogName, Catalog.Type.FILESET, @@ -211,7 +362,7 @@ private String defaultBaseLocation() { defaultBaseLocation = String.format( "hdfs://%s:%d/user/hadoop/%s", - containerSuite.getHiveContainer().getContainerIpAddress(), + containerSuite.getHiveRangerContainer().getContainerIpAddress(), HiveContainer.HDFS_DEFAULTFS_PORT, schemaName.toLowerCase()); } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index cb41e79216c..600463fbc21 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -67,7 +67,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(); + RangerITEnv.init(true); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index 356e34d2824..9c45a21099e 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -80,7 +80,7 @@ public class RangerHiveIT { @BeforeAll public static void setup() { - RangerITEnv.init(); + RangerITEnv.init(true); rangerAuthHivePlugin = RangerITEnv.rangerAuthHivePlugin; rangerHelper = RangerITEnv.rangerHelper; diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 46703cf1b7b..ff08ed28a20 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -87,7 +87,7 @@ public class RangerITEnv { protected static RangerHelper rangerHDFSHelper; - public static void init() { + public static void init(boolean allowAnyoneAccessHDFS) { containerSuite.startRangerContainer(); rangerClient = containerSuite.getRangerContainer().rangerClient; @@ -148,7 +148,9 @@ public static void init() { // No IP address set, no impact on testing createRangerHdfsRepository("", true); createRangerHiveRepository("", true); - allowAnyoneAccessHDFS(); + if (allowAnyoneAccessHDFS) { + allowAnyoneAccessHDFS(); + } initRangerService = true; } } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java index 7b45eda7a6e..a4fc1253efe 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java @@ -71,7 +71,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(); + RangerITEnv.init(true); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java index 7cb600b9d8c..b2529837e3c 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java @@ -70,7 +70,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.init(); + RangerITEnv.init(true); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = From 1618cda8f0dba59e5d6db629a4e1643803c88393 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Mon, 9 Dec 2024 08:25:53 +0800 Subject: [PATCH 19/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../test/RangerAuthorizationHDFSPluginIT.java | 21 +++++++++++++------ .../test/RangerAuthorizationPluginIT.java | 6 ++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index 9ffbf89d433..933ead9e388 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -31,6 +31,7 @@ import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.authorization.ranger.RangerHDFSMetadataObject; import org.apache.gravitino.authorization.ranger.RangerPrivileges; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -47,22 +48,30 @@ public static void setup() { rangerAuthPlugin = RangerITEnv.rangerAuthHDFSPlugin; } + @AfterAll + public static void cleanup() { + RangerITEnv.cleanup(); + } + @Test public void testTranslateMetadataObject() { MetadataObject metalake = MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); - Assertions.assertThrows( - IllegalArgumentException.class, () -> rangerAuthPlugin.translateMetadataObject(metalake)); + Assertions.assertEquals( + RangerHDFSMetadataObject.Type.SCHEMA_PATH, + rangerAuthPlugin.translateMetadataObject(metalake).type()); MetadataObject catalog = MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); - Assertions.assertThrows( - IllegalArgumentException.class, () -> rangerAuthPlugin.translateMetadataObject(catalog)); + Assertions.assertEquals( + RangerHDFSMetadataObject.Type.SCHEMA_PATH, + rangerAuthPlugin.translateMetadataObject(catalog).type()); MetadataObject schema = MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); - Assertions.assertThrows( - IllegalArgumentException.class, () -> rangerAuthPlugin.translateMetadataObject(schema)); + Assertions.assertEquals( + RangerHDFSMetadataObject.Type.SCHEMA_PATH, + rangerAuthPlugin.translateMetadataObject(schema).type()); MetadataObject table = MetadataObjects.parse(String.format("catalog1.schema1.tab1"), MetadataObject.Type.TABLE); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java index f3a474f213d..74ddf078491 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java @@ -33,6 +33,7 @@ import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.authorization.ranger.RangerHadoopSQLMetadataObject; import org.apache.gravitino.authorization.ranger.RangerHelper; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -48,6 +49,11 @@ public static void setup() { rangerAuthPlugin = RangerITEnv.rangerAuthHivePlugin; } + @AfterAll + public static void cleanup() { + RangerITEnv.cleanup(); + } + @Test public void testTranslateMetadataObject() { MetadataObject metalake = From a0f58c951d8a9ebe0b7c56526d3b853865143c6b Mon Sep 17 00:00:00 2001 From: theoryxu Date: Mon, 9 Dec 2024 11:24:45 +0800 Subject: [PATCH 20/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../integration/test/RangerFilesetIT.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index b9aa93a707a..c9ceeba0784 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -64,6 +64,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -75,6 +76,7 @@ public class RangerFilesetIT extends BaseIT { private String RANGER_ADMIN_URL; private String defaultBaseLocation; + private static final String HADOOP_USER_NAME = "HADOOP_USER_NAME"; private String metalakeName = "metalake"; private String catalogName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_catalog"); private String schemaName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_schema"); @@ -136,6 +138,7 @@ public void stop() throws IOException { } @Test + @Order(0) void testReadWritePath() throws IOException, RangerServiceException { String filename = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_fileset"); Fileset fileset = @@ -319,6 +322,40 @@ void testReadWritePath() throws IOException, RangerServiceException { Assertions.assertEquals(3, policies.get(0).getPolicyItems().size()); } + @Test + @Order(1) + void testReadWritePathE2E() throws IOException, RangerServiceException, InterruptedException { + String filenameRole = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_fileset"); + Fileset fileset = + catalog + .asFilesetCatalog() + .createFileset( + NameIdentifier.of(schemaName, filenameRole), + "comment", + Fileset.Type.MANAGED, + storageLocation(filenameRole), + null); + Assertions.assertTrue( + catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, fileset.name()))); + Assertions.assertTrue(fileSystem.exists(new Path(storageLocation(filenameRole)))); + + String filesetRole = currentFunName() + "_testReadWritePathE2E"; + SecurableObject securableObject = + SecurableObjects.parse( + String.format("%s.%s.%s", catalogName, schemaName, fileset.name()), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.ReadFileset.allow())); + metalake.createRole(filesetRole, Collections.emptyMap(), Lists.newArrayList(securableObject)); + String userName1 = System.getenv(HADOOP_USER_NAME); + metalake.addUser(userName1); + metalake.grantRolesToUser(Lists.newArrayList(filesetRole), userName1); + waitForUpdatingPolicies(); + Assertions.assertDoesNotThrow( + () -> fileSystem.listFiles(new Path(storageLocation(filenameRole)), false)); + Assertions.assertThrows( + Exception.class, () -> fileSystem.mkdirs(new Path(storageLocation(filenameRole)))); + } + private void createCatalogAndSchema() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); From ec234c3f685a855d54e350b82c85797652052951 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 11 Dec 2024 12:07:20 +0800 Subject: [PATCH 21/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 4 +- .../integration/test/RangerFilesetIT.java | 127 ++++++++++++++++-- 2 files changed, 121 insertions(+), 10 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 4273723eece..454752b9c53 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -74,7 +74,9 @@ public static synchronized RangerAuthorizationHDFSPlugin getInstance( public Map> privilegesMappingRule() { return ImmutableMap.of( Privilege.Name.READ_FILESET, - ImmutableSet.of(RangerPrivileges.RangerHdfsPrivilege.READ), + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), Privilege.Name.WRITE_FILESET, ImmutableSet.of( RangerPrivileges.RangerHdfsPrivilege.WRITE, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index c9ceeba0784..c3c9817bc41 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -33,6 +33,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -59,6 +60,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.RangerServiceException; import org.apache.ranger.plugin.model.RangerPolicy; import org.junit.jupiter.api.AfterAll; @@ -76,7 +79,6 @@ public class RangerFilesetIT extends BaseIT { private String RANGER_ADMIN_URL; private String defaultBaseLocation; - private static final String HADOOP_USER_NAME = "HADOOP_USER_NAME"; private String metalakeName = "metalake"; private String catalogName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_catalog"); private String schemaName = GravitinoITUtils.genRandomName("RangerFilesetE2EIT_schema"); @@ -218,7 +220,7 @@ void testReadWritePath() throws IOException, RangerServiceException { .anyMatch(access -> access.getType().equals("write"))) .count()); Assertions.assertEquals( - 0, + 1, policies.get(0).getPolicyItems().stream() .filter( item -> @@ -338,6 +340,32 @@ void testReadWritePathE2E() throws IOException, RangerServiceException, Interrup Assertions.assertTrue( catalog.asFilesetCatalog().filesetExists(NameIdentifier.of(schemaName, fileset.name()))); Assertions.assertTrue(fileSystem.exists(new Path(storageLocation(filenameRole)))); + FsPermission permission = new FsPermission("700"); + fileSystem.setPermission(new Path(storageLocation(filenameRole)), permission); + + String userName = "userTestReadWritePathE2E"; + metalake.addUser(userName); + + UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser()) + .doAs( + (PrivilegedExceptionAction) + () -> { + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", defaultBaseLocation()); + FileSystem userFileSystem = FileSystem.get(conf); + Assertions.assertThrows( + Exception.class, + () -> + userFileSystem.listFiles(new Path(storageLocation(filenameRole)), false)); + Assertions.assertThrows( + Exception.class, + () -> + userFileSystem.mkdirs( + new Path( + String.format("%s/%s", storageLocation(filenameRole), "test1")))); + userFileSystem.close(); + return null; + }); String filesetRole = currentFunName() + "_testReadWritePathE2E"; SecurableObject securableObject = @@ -346,14 +374,95 @@ void testReadWritePathE2E() throws IOException, RangerServiceException, Interrup MetadataObject.Type.FILESET, Lists.newArrayList(Privileges.ReadFileset.allow())); metalake.createRole(filesetRole, Collections.emptyMap(), Lists.newArrayList(securableObject)); - String userName1 = System.getenv(HADOOP_USER_NAME); - metalake.addUser(userName1); - metalake.grantRolesToUser(Lists.newArrayList(filesetRole), userName1); + metalake.grantRolesToUser(Lists.newArrayList(filesetRole), userName); + waitForUpdatingPolicies(); + + UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser()) + .doAs( + (PrivilegedExceptionAction) + () -> { + FileSystem userFileSystem = + FileSystem.get( + new Configuration() { + { + set("fs.defaultFS", defaultBaseLocation()); + } + }); + Assertions.assertDoesNotThrow( + () -> + userFileSystem.listFiles(new Path(storageLocation(filenameRole)), false)); + Assertions.assertThrows( + Exception.class, + () -> + userFileSystem.mkdirs( + new Path( + String.format("%s/%s", storageLocation(filenameRole), "test2")))); + userFileSystem.close(); + return null; + }); + + MetadataObject filesetObject = + MetadataObjects.of( + String.format("%s.%s", catalogName, schemaName), + fileset.name(), + MetadataObject.Type.FILESET); + metalake.grantPrivilegesToRole( + filesetRole, filesetObject, Lists.newArrayList(Privileges.WriteFileset.allow())); waitForUpdatingPolicies(); - Assertions.assertDoesNotThrow( - () -> fileSystem.listFiles(new Path(storageLocation(filenameRole)), false)); - Assertions.assertThrows( - Exception.class, () -> fileSystem.mkdirs(new Path(storageLocation(filenameRole)))); + UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser()) + .doAs( + (PrivilegedExceptionAction) + () -> { + FileSystem userFileSystem = + FileSystem.get( + new Configuration() { + { + set("fs.defaultFS", defaultBaseLocation()); + } + }); + Assertions.assertDoesNotThrow( + () -> + userFileSystem.listFiles(new Path(storageLocation(filenameRole)), false)); + Assertions.assertDoesNotThrow( + () -> + userFileSystem.mkdirs( + new Path( + String.format("%s/%s", storageLocation(filenameRole), "test3")))); + userFileSystem.close(); + return null; + }); + + metalake.revokePrivilegesFromRole( + filesetRole, + filesetObject, + Lists.newArrayList(Privileges.ReadFileset.allow(), Privileges.WriteFileset.allow())); + waitForUpdatingPolicies(); + UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser()) + .doAs( + (PrivilegedExceptionAction) + () -> { + FileSystem userFileSystem = + FileSystem.get( + new Configuration() { + { + set("fs.defaultFS", defaultBaseLocation()); + } + }); + Assertions.assertThrows( + Exception.class, + () -> + userFileSystem.listFiles(new Path(storageLocation(filenameRole)), false)); + Assertions.assertThrows( + Exception.class, + () -> + userFileSystem.mkdirs( + new Path( + String.format("%s/%s", storageLocation(filenameRole), "test4")))); + userFileSystem.close(); + return null; + }); + + catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(schemaName, fileset.name())); } private void createCatalogAndSchema() { From df5c91f7416b998f30c51367dddf1eae5ab586bf Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 11 Dec 2024 19:03:03 +0800 Subject: [PATCH 22/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../test/RangerAuthorizationHDFSPluginIT.java | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index 933ead9e388..0210e3578bc 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -141,24 +141,7 @@ public void testTranslatePrivilege() { securableObject -> { Assertions.assertEquals(RangerHDFSMetadataObject.Type.PATH, securableObject.type()); Assertions.assertEquals("/test", securableObject.fullName()); - Assertions.assertTrue( - securableObject.privileges().size() == 1 || securableObject.privileges().size() == 2); - if (securableObject.privileges().size() == 1) { - Assertions.assertEquals( - RangerPrivileges.RangerHdfsPrivilege.READ.getName(), - securableObject.privileges().get(0).getName()); - } else { - securableObject - .privileges() - .forEach( - privilege -> { - Assertions.assertTrue( - ImmutableList.of( - RangerPrivileges.RangerHdfsPrivilege.WRITE.getName(), - RangerPrivileges.RangerHdfsPrivilege.EXECUTE.getName()) - .contains(privilege.getName())); - }); - } + Assertions.assertEquals(2, securableObject.privileges().size()); }); } From e767e21ac45bc649181d6f46b04e9c7604dd82da Mon Sep 17 00:00:00 2001 From: theoryxu Date: Wed, 11 Dec 2024 20:00:37 +0800 Subject: [PATCH 23/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../integration/test/RangerAuthorizationHDFSPluginIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index 0210e3578bc..9f7f86d9c21 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.authorization.ranger.integration.test; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.List; import org.apache.gravitino.MetadataObject; @@ -30,7 +29,6 @@ import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.authorization.ranger.RangerHDFSMetadataObject; -import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; From 5a44c8825e45d3f94f612dd57b20b2a565089d40 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 12 Dec 2024 10:11:39 +0800 Subject: [PATCH 24/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 5 +- .../ranger/RangerAuthorizationPlugin.java | 52 ++++----- .../ranger/RangerHDFSMetadataObject.java | 3 +- .../test/RangerAuthorizationHDFSPluginIT.java | 6 +- .../integration/test/RangerFilesetIT.java | 102 ++++++++++++------ 5 files changed, 97 insertions(+), 71 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 454752b9c53..dbe53c1204d 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -233,14 +233,13 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); return rangerHDFSMetadataObject; } else { - return new RangerHDFSMetadataObject("", RangerHDFSMetadataObject.Type.SCHEMA_PATH); + return new RangerHDFSMetadataObject("", RangerHDFSMetadataObject.Type.PATH); } } private String getFileSetPath(MetadataObject metadataObject) { FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - boolean testEnv = - System.getenv("GRAVITINO_TEST") != null || System.getenv("GRAVITINO_TEST_CLOUD_IT") == null; + boolean testEnv = System.getenv("GRAVITINO_TEST") != null; if (filesetDispatcher == null && testEnv) { return "/test"; } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index baaa4beff65..afe74774bfc 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -328,11 +328,9 @@ public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws Runtime } else if (change instanceof MetadataObjectChange.RemoveMetadataObject) { MetadataObject metadataObject = ((MetadataObjectChange.RemoveMetadataObject) change).metadataObject(); - if (metadataObject.type() != MetadataObject.Type.FILESET) { - AuthorizationMetadataObject AuthorizationMetadataObject = - translateMetadataObject(metadataObject); - doRemoveMetadataObject(AuthorizationMetadataObject); - } + AuthorizationMetadataObject AuthorizationMetadataObject = + translateMetadataObject(metadataObject); + doRemoveMetadataObject(AuthorizationMetadataObject); } else { throw new IllegalArgumentException( "Unsupported metadata object change type: " @@ -848,34 +846,22 @@ private void removePolicyItemIfEqualRoleName( * IF remove the COLUMN, Only need to remove `{schema}.*.*`
*/ private void doRemoveMetadataObject(AuthorizationMetadataObject authMetadataObject) { - if (authMetadataObject instanceof RangerHadoopSQLMetadataObject) { - switch (authMetadataObject.metadataObjectType()) { - case SCHEMA: - doRemoveSchemaMetadataObject(authMetadataObject); - break; - case TABLE: - doRemoveTableMetadataObject(authMetadataObject); - break; - case COLUMN: - removePolicyByMetadataObject(authMetadataObject.names()); - break; - default: - throw new IllegalArgumentException( - "Unsupported metadata object type: " + authMetadataObject.type()); - } - } else if (authMetadataObject instanceof RangerHDFSMetadataObject) { - switch (authMetadataObject.metadataObjectType()) { - case FILESET: - removePolicyByMetadataObject(authMetadataObject.names()); - break; - default: - LOG.info( - "type {} do nothing in RangerHDFSMetadataObject", - authMetadataObject.metadataObjectType()); - } - } else { - throw new IllegalArgumentException( - "Unsupported authorization Metadata object: " + authMetadataObject); + switch (authMetadataObject.metadataObjectType()) { + case SCHEMA: + doRemoveSchemaMetadataObject(authMetadataObject); + break; + case TABLE: + doRemoveTableMetadataObject(authMetadataObject); + break; + case COLUMN: + removePolicyByMetadataObject(authMetadataObject.names()); + break; + case FILESET: + // can not get fileset path in this case, do nothing + break; + default: + throw new IllegalArgumentException( + "Unsupported metadata object type: " + authMetadataObject.type()); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java index 419382083b4..2bf842dc9ce 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java @@ -32,8 +32,7 @@ public class RangerHDFSMetadataObject implements AuthorizationMetadataObject { */ public enum Type implements AuthorizationMetadataObject.Type { /** A path is mapped the path of storages like HDFS, S3 etc. */ - PATH(MetadataObject.Type.FILESET), - SCHEMA_PATH(MetadataObject.Type.SCHEMA); + PATH(MetadataObject.Type.FILESET); private final MetadataObject.Type metadataType; Type(MetadataObject.Type type) { diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index 9f7f86d9c21..c9e7209d001 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -56,19 +56,19 @@ public void testTranslateMetadataObject() { MetadataObject metalake = MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); Assertions.assertEquals( - RangerHDFSMetadataObject.Type.SCHEMA_PATH, + RangerHDFSMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(metalake).type()); MetadataObject catalog = MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); Assertions.assertEquals( - RangerHDFSMetadataObject.Type.SCHEMA_PATH, + RangerHDFSMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(catalog).type()); MetadataObject schema = MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); Assertions.assertEquals( - RangerHDFSMetadataObject.Type.SCHEMA_PATH, + RangerHDFSMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(schema).type()); MetadataObject table = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index c3c9817bc41..bbaae32781b 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -50,6 +50,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerHelper; +import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.connector.AuthorizationPropertiesMeta; import org.apache.gravitino.file.Fileset; @@ -166,7 +167,12 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter(item -> item.getRoles().contains(RangerHelper.GRAVITINO_OWNER_ROLE)) .filter( item -> - item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + item.getAccesses().stream() + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.READ.getName()))) .count()); Assertions.assertEquals( 1, @@ -175,7 +181,11 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("write"))) + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.WRITE.getName()))) .count()); Assertions.assertEquals( 1, @@ -184,7 +194,12 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("execute"))) + .anyMatch( + access -> + access + .getType() + .equals( + RangerPrivileges.RangerHdfsPrivilege.EXECUTE.getName()))) .count()); String filesetRole = currentFunName(); @@ -206,7 +221,12 @@ void testReadWritePath() throws IOException, RangerServiceException { item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) .filter( item -> - item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + item.getAccesses().stream() + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.READ.getName()))) .count()); Assertions.assertEquals( 0, @@ -217,7 +237,11 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("write"))) + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.WRITE.getName()))) .count()); Assertions.assertEquals( 1, @@ -228,7 +252,12 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("execute"))) + .anyMatch( + access -> + access + .getType() + .equals( + RangerPrivileges.RangerHdfsPrivilege.EXECUTE.getName()))) .count()); metalake.grantPrivilegesToRole( @@ -250,7 +279,12 @@ void testReadWritePath() throws IOException, RangerServiceException { item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) .filter( item -> - item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + item.getAccesses().stream() + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.READ.getName()))) .count()); Assertions.assertEquals( 1, @@ -261,7 +295,11 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("write"))) + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.WRITE.getName()))) .count()); Assertions.assertEquals( 1, @@ -272,7 +310,12 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("execute"))) + .anyMatch( + access -> + access + .getType() + .equals( + RangerPrivileges.RangerHdfsPrivilege.EXECUTE.getName()))) .count()); metalake.revokePrivilegesFromRole( @@ -293,7 +336,12 @@ void testReadWritePath() throws IOException, RangerServiceException { item.getRoles().contains(rangerHelper.generateGravitinoRoleName(filesetRole))) .filter( item -> - item.getAccesses().stream().anyMatch(access -> access.getType().equals("read"))) + item.getAccesses().stream() + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.READ.getName()))) .count()); Assertions.assertEquals( 0, @@ -304,7 +352,11 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("write"))) + .anyMatch( + access -> + access + .getType() + .equals(RangerPrivileges.RangerHdfsPrivilege.WRITE.getName()))) .count()); Assertions.assertEquals( 0, @@ -315,7 +367,12 @@ void testReadWritePath() throws IOException, RangerServiceException { .filter( item -> item.getAccesses().stream() - .anyMatch(access -> access.getType().equals("execute"))) + .anyMatch( + access -> + access + .getType() + .equals( + RangerPrivileges.RangerHdfsPrivilege.EXECUTE.getName()))) .count()); catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(schemaName, fileset.name())); @@ -375,7 +432,7 @@ void testReadWritePathE2E() throws IOException, RangerServiceException, Interrup Lists.newArrayList(Privileges.ReadFileset.allow())); metalake.createRole(filesetRole, Collections.emptyMap(), Lists.newArrayList(securableObject)); metalake.grantRolesToUser(Lists.newArrayList(filesetRole), userName); - waitForUpdatingPolicies(); + RangerBaseE2EIT.waitForUpdatingPolicies(); UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser()) .doAs( @@ -408,7 +465,7 @@ void testReadWritePathE2E() throws IOException, RangerServiceException, Interrup MetadataObject.Type.FILESET); metalake.grantPrivilegesToRole( filesetRole, filesetObject, Lists.newArrayList(Privileges.WriteFileset.allow())); - waitForUpdatingPolicies(); + RangerBaseE2EIT.waitForUpdatingPolicies(); UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser()) .doAs( (PrivilegedExceptionAction) @@ -436,7 +493,7 @@ void testReadWritePathE2E() throws IOException, RangerServiceException, Interrup filesetRole, filesetObject, Lists.newArrayList(Privileges.ReadFileset.allow(), Privileges.WriteFileset.allow())); - waitForUpdatingPolicies(); + RangerBaseE2EIT.waitForUpdatingPolicies(); UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser()) .doAs( (PrivilegedExceptionAction) @@ -515,22 +572,7 @@ private String defaultBaseLocation() { return defaultBaseLocation; } - private Fileset createFileset(String filesetName, Fileset.Type type, String storageLocation) { - return catalog - .asFilesetCatalog() - .createFileset( - NameIdentifier.of(schemaName, filesetName), "comment", type, storageLocation, null); - } - private String storageLocation(String filesetName) { return defaultBaseLocation() + "/" + filesetName; } - - private void waitForUpdatingPolicies() throws InterruptedException { - // After Ranger authorization, Must wait a period of time for the Ranger Spark plugin to update - // the policy Sleep time must be greater than the policy update interval - // (ranger.plugin.spark.policy.pollIntervalMs) in the - // `resources/ranger-spark-security.xml.template` - Thread.sleep(1000L); - } } From 69944a425f4e83203934777b44bd1cc7291af0b6 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 12 Dec 2024 10:37:11 +0800 Subject: [PATCH 25/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../authorization/ranger/RangerAuthorizationPlugin.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index afe74774bfc..a3ce047aa5b 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -328,9 +328,11 @@ public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws Runtime } else if (change instanceof MetadataObjectChange.RemoveMetadataObject) { MetadataObject metadataObject = ((MetadataObjectChange.RemoveMetadataObject) change).metadataObject(); - AuthorizationMetadataObject AuthorizationMetadataObject = - translateMetadataObject(metadataObject); - doRemoveMetadataObject(AuthorizationMetadataObject); + if (metadataObject.type() != MetadataObject.Type.FILESET) { + AuthorizationMetadataObject AuthorizationMetadataObject = + translateMetadataObject(metadataObject); + doRemoveMetadataObject(AuthorizationMetadataObject); + } } else { throw new IllegalArgumentException( "Unsupported metadata object change type: " From 364e0efc01c3dd0ef4ff116bcd6dfb2dab85aee4 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 12 Dec 2024 11:12:47 +0800 Subject: [PATCH 26/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../authorization/ranger/RangerAuthorizationHDFSPlugin.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index dbe53c1204d..bfb1c66fd57 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -239,7 +239,8 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada private String getFileSetPath(MetadataObject metadataObject) { FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - boolean testEnv = System.getenv("GRAVITINO_TEST") != null; + boolean testEnv = + System.getenv("GRAVITINO_TEST") != null || System.getenv("GRAVITINO_TEST_CLOUD_IT") == null; if (filesetDispatcher == null && testEnv) { return "/test"; } From ec8343b1e82c8bae4e1d59c588bbabd7e45b7e01 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 12 Dec 2024 15:49:36 +0800 Subject: [PATCH 27/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- .../ranger/RangerAuthorizationHDFSPlugin.java | 16 ++++++++-------- ...ct.java => RangerPathBaseMetadataObject.java} | 6 +++--- ...t.java => RangerPathBaseSecurableObject.java} | 4 ++-- .../test/RangerAuthorizationHDFSPluginIT.java | 14 +++++++------- 4 files changed, 20 insertions(+), 20 deletions(-) rename authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/{RangerHDFSMetadataObject.java => RangerPathBaseMetadataObject.java} (92%) rename authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/{RangerHDFSSecurableObject.java => RangerPathBaseSecurableObject.java} (92%) diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index bfb1c66fd57..c564b39b953 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -113,9 +113,9 @@ public AuthorizationSecurableObject generateAuthorizationSecurableObject( AuthorizationMetadataObject.Type type, Set privileges) { AuthorizationMetadataObject authMetadataObject = - new RangerHDFSMetadataObject(AuthorizationMetadataObject.getLastName(names), type); + new RangerPathBaseMetadataObject(AuthorizationMetadataObject.getLastName(names), type); authMetadataObject.validateAuthorizationMetadataObject(); - return new RangerHDFSSecurableObject( + return new RangerPathBaseSecurableObject( authMetadataObject.name(), authMetadataObject.type(), privileges); } @@ -170,7 +170,7 @@ public List translatePrivilege(SecurableObject sec rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), - RangerHDFSMetadataObject.Type.PATH, + RangerPathBaseMetadataObject.Type.PATH, rangerPrivileges)); break; default: @@ -202,7 +202,7 @@ public List translateOwner(MetadataObject gravitin rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), - RangerHDFSMetadataObject.Type.PATH, + RangerPathBaseMetadataObject.Type.PATH, ownerMappingRule())); break; default: @@ -227,13 +227,13 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada nsMetadataObject.size() > 0, "The metadata object must have at least one name."); if (metadataObject.type() == MetadataObject.Type.FILESET) { - RangerHDFSMetadataObject rangerHDFSMetadataObject = - new RangerHDFSMetadataObject( - getFileSetPath(metadataObject), RangerHDFSMetadataObject.Type.PATH); + RangerPathBaseMetadataObject rangerHDFSMetadataObject = + new RangerPathBaseMetadataObject( + getFileSetPath(metadataObject), RangerPathBaseMetadataObject.Type.PATH); rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); return rangerHDFSMetadataObject; } else { - return new RangerHDFSMetadataObject("", RangerHDFSMetadataObject.Type.PATH); + return new RangerPathBaseMetadataObject("", RangerPathBaseMetadataObject.Type.PATH); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java similarity index 92% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java rename to authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java index 2bf842dc9ce..77523464162 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java @@ -25,7 +25,7 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.AuthorizationMetadataObject; -public class RangerHDFSMetadataObject implements AuthorizationMetadataObject { +public class RangerPathBaseMetadataObject implements AuthorizationMetadataObject { /** * The type of object in the Ranger system. Every type will map one kind of the entity of the * Gravitino type system. @@ -59,7 +59,7 @@ public static RangerHadoopSQLMetadataObject.Type fromMetadataType( private final AuthorizationMetadataObject.Type type; - public RangerHDFSMetadataObject(String path, AuthorizationMetadataObject.Type type) { + public RangerPathBaseMetadataObject(String path, AuthorizationMetadataObject.Type type) { this.path = path; this.type = type; } @@ -97,7 +97,7 @@ public void validateAuthorizationMetadataObject() throws IllegalArgumentExceptio type != null, "Cannot create a Ranger metadata object with no type"); Preconditions.checkArgument( - type == RangerHDFSMetadataObject.Type.PATH, "it must be the PATH type"); + type == RangerPathBaseMetadataObject.Type.PATH, "it must be the PATH type"); for (String name : names) { Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSSecurableObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java similarity index 92% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSSecurableObject.java rename to authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java index df1bac73545..bd2c73fdaef 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHDFSSecurableObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java @@ -25,12 +25,12 @@ import org.apache.gravitino.authorization.AuthorizationPrivilege; import org.apache.gravitino.authorization.AuthorizationSecurableObject; -public class RangerHDFSSecurableObject extends RangerHDFSMetadataObject +public class RangerPathBaseSecurableObject extends RangerPathBaseMetadataObject implements AuthorizationSecurableObject { private final List privileges; - public RangerHDFSSecurableObject( + public RangerPathBaseSecurableObject( String path, AuthorizationMetadataObject.Type type, Set privileges) { super(path, type); this.privileges = ImmutableList.copyOf(privileges); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index c9e7209d001..e1eacba1587 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -28,7 +28,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; -import org.apache.gravitino.authorization.ranger.RangerHDFSMetadataObject; +import org.apache.gravitino.authorization.ranger.RangerPathBaseMetadataObject; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -56,19 +56,19 @@ public void testTranslateMetadataObject() { MetadataObject metalake = MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); Assertions.assertEquals( - RangerHDFSMetadataObject.Type.PATH, + RangerPathBaseMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(metalake).type()); MetadataObject catalog = MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); Assertions.assertEquals( - RangerHDFSMetadataObject.Type.PATH, + RangerPathBaseMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(catalog).type()); MetadataObject schema = MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); Assertions.assertEquals( - RangerHDFSMetadataObject.Type.PATH, + RangerPathBaseMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(schema).type()); MetadataObject table = @@ -82,7 +82,7 @@ public void testTranslateMetadataObject() { AuthorizationMetadataObject rangerFileset = rangerAuthPlugin.translateMetadataObject(fileset); Assertions.assertEquals(1, rangerFileset.names().size()); Assertions.assertEquals("/test", rangerFileset.fullName()); - Assertions.assertEquals(RangerHDFSMetadataObject.Type.PATH, rangerFileset.type()); + Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, rangerFileset.type()); } @Test @@ -137,7 +137,7 @@ public void testTranslatePrivilege() { filesetInFileset1.forEach( securableObject -> { - Assertions.assertEquals(RangerHDFSMetadataObject.Type.PATH, securableObject.type()); + Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, securableObject.type()); Assertions.assertEquals("/test", securableObject.fullName()); Assertions.assertEquals(2, securableObject.privileges().size()); }); @@ -166,7 +166,7 @@ public void testTranslateOwner() { List filesetOwner = rangerAuthPlugin.translateOwner(fileset); Assertions.assertEquals(1, filesetOwner.size()); Assertions.assertEquals("/test", filesetOwner.get(0).fullName()); - Assertions.assertEquals(RangerHDFSMetadataObject.Type.PATH, filesetOwner.get(0).type()); + Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, filesetOwner.get(0).type()); Assertions.assertEquals(3, filesetOwner.get(0).privileges().size()); } } From f2998a523d2ac8827595822718c836c3d114df77 Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 12 Dec 2024 16:09:06 +0800 Subject: [PATCH 28/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- authorizations/authorization-ranger/build.gradle.kts | 1 + .../authorization/ranger/RangerAuthorizationHDFSPlugin.java | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index f83aee72c54..4071fce74da 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -132,6 +132,7 @@ tasks { tasks.test { doFirst { environment("HADOOP_USER_NAME", "gravitino") + environment("GRAVITINO_TEST", "true") } dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars", ":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars", ":catalogs:catalog-lakehouse-paimon:jar", ":catalogs:catalog-lakehouse-paimon:runtimeJars") diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index c564b39b953..ada1cdaa6bc 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -239,8 +239,7 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada private String getFileSetPath(MetadataObject metadataObject) { FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - boolean testEnv = - System.getenv("GRAVITINO_TEST") != null || System.getenv("GRAVITINO_TEST_CLOUD_IT") == null; + boolean testEnv = System.getenv("GRAVITINO_TEST") != null; if (filesetDispatcher == null && testEnv) { return "/test"; } From b13fc7c7fafe6760ec6e2f35b2b98389752a36db Mon Sep 17 00:00:00 2001 From: theoryxu Date: Thu, 12 Dec 2024 18:12:35 +0800 Subject: [PATCH 29/31] [apache#5731]feat(auth-ranger): RangerAuthorizationHDFSPlugin supports Fileset authorization --- authorizations/authorization-ranger/build.gradle.kts | 1 - .../ranger/RangerAuthorizationHDFSPlugin.java | 12 ++++++++++-- .../ranger/integration/test/RangerITEnv.java | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index 4071fce74da..f83aee72c54 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -132,7 +132,6 @@ tasks { tasks.test { doFirst { environment("HADOOP_USER_NAME", "gravitino") - environment("GRAVITINO_TEST", "true") } dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars", ":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars", ":catalogs:catalog-lakehouse-paimon:jar", ":catalogs:catalog-lakehouse-paimon:runtimeJars") diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index ada1cdaa6bc..7848a3eb67a 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -54,6 +54,8 @@ public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { private static volatile RangerAuthorizationHDFSPlugin instance = null; + private boolean isTestEnv = false; + private RangerAuthorizationHDFSPlugin(String metalake, Map config) { super(metalake, config); } @@ -70,6 +72,13 @@ public static synchronized RangerAuthorizationHDFSPlugin getInstance( return instance; } + public static synchronized RangerAuthorizationHDFSPlugin getInstanceForTest( + String metalake, Map config) { + getInstance(metalake, config); + instance.isTestEnv = true; + return instance; + } + @Override public Map> privilegesMappingRule() { return ImmutableMap.of( @@ -239,8 +248,7 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada private String getFileSetPath(MetadataObject metadataObject) { FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - boolean testEnv = System.getenv("GRAVITINO_TEST") != null; - if (filesetDispatcher == null && testEnv) { + if (filesetDispatcher == null && isTestEnv) { return "/test"; } NameIdentifier identifier = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index ff08ed28a20..abfe19bc89b 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -110,7 +110,7 @@ public static void init(boolean allowAnyoneAccessHDFS) { RangerITEnv.RANGER_HIVE_REPO_NAME)); rangerAuthHDFSPlugin = - RangerAuthorizationHDFSPlugin.getInstance( + RangerAuthorizationHDFSPlugin.getInstanceForTest( "metalake", ImmutableMap.of( AuthorizationPropertiesMeta.RANGER_ADMIN_URL, From 49ff945626e7583f8348662c6a10cdddf2be9630 Mon Sep 17 00:00:00 2001 From: Xun Date: Thu, 12 Dec 2024 18:43:14 +0800 Subject: [PATCH 30/31] test --- authorizations/authorization-ranger/build.gradle.kts | 3 ++- .../ranger/RangerAuthorizationHDFSPlugin.java | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index f83aee72c54..8bc1eba6929 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -131,9 +131,10 @@ tasks { tasks.test { doFirst { + environment("GRAVITINO_TEST", "true") environment("HADOOP_USER_NAME", "gravitino") } - dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars", ":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars", ":catalogs:catalog-lakehouse-paimon:jar", ":catalogs:catalog-lakehouse-paimon:runtimeJars") + dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars", ":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars", ":catalogs:catalog-lakehouse-paimon:jar", ":catalogs:catalog-lakehouse-paimon:runtimeJars", ":catalogs:catalog-hadoop:jar", ":catalogs:catalog-hadoop:runtimeJars") val skipITs = project.hasProperty("skipITs") if (skipITs) { diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 7848a3eb67a..16cd8466e12 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -54,8 +54,6 @@ public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { private static volatile RangerAuthorizationHDFSPlugin instance = null; - private boolean isTestEnv = false; - private RangerAuthorizationHDFSPlugin(String metalake, Map config) { super(metalake, config); } @@ -75,7 +73,6 @@ public static synchronized RangerAuthorizationHDFSPlugin getInstance( public static synchronized RangerAuthorizationHDFSPlugin getInstanceForTest( String metalake, Map config) { getInstance(metalake, config); - instance.isTestEnv = true; return instance; } @@ -248,7 +245,9 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada private String getFileSetPath(MetadataObject metadataObject) { FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - if (filesetDispatcher == null && isTestEnv) { + boolean testEnv = System.getenv("GRAVITINO_TEST") != null; + LOG.debug("testEnv: {}", testEnv); + if (filesetDispatcher == null && testEnv) { return "/test"; } NameIdentifier identifier = From 1aec3b2fe49c9e28c98951e1648377469f0153c9 Mon Sep 17 00:00:00 2001 From: Xun Date: Fri, 13 Dec 2024 11:04:11 +0800 Subject: [PATCH 31/31] debug action --- .github/workflows/access-control-integration-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index 54ffde2ee82..92d3e53ebda 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -84,6 +84,10 @@ jobs: run: | dev/ci/util_free_space.sh + - name: Setup debug Github Action + if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} + uses: csexton/debugger-action@master + - name: Authorization Integration Test (JDK${{ matrix.java-version }}) id: integrationTest run: |