diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index ad347aa7e60..54ffde2ee82 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -87,9 +87,9 @@ jobs: - name: Authorization Integration Test (JDK${{ matrix.java-version }}) id: integrationTest run: | - ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.**" - ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.**" - ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.**" + ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test + ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test + ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/api/src/main/java/org/apache/gravitino/authorization/MetadataObjectChange.java b/api/src/main/java/org/apache/gravitino/authorization/MetadataObjectChange.java new file mode 100644 index 00000000000..a7281d97d5a --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/authorization/MetadataObjectChange.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization; + +import com.google.common.base.Preconditions; +import java.util.Objects; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.annotation.Evolving; + +/** + * The MetadataObjectChange interface defines the public API for managing roles in an authorization. + */ +@Evolving +public interface MetadataObjectChange { + /** + * Rename a metadata entity MetadataObjectChange. + * + * @param metadataObject The metadata object. + * @param newMetadataObject The new metadata object. + * @return return a MetadataObjectChange for the rename metadata object. + */ + static MetadataObjectChange rename( + MetadataObject metadataObject, MetadataObject newMetadataObject) { + return new RenameMetadataObject(metadataObject, newMetadataObject); + } + + /** + * Remove a metadata entity MetadataObjectChange. + * + * @param metadataObject The metadata object. + * @return return a MetadataObjectChange for the remove metadata object. + */ + static MetadataObjectChange remove(MetadataObject metadataObject) { + return new RemoveMetadataObject(metadataObject); + } + + /** A RenameMetadataObject is to rename securable object's metadata entity. */ + final class RenameMetadataObject implements MetadataObjectChange { + private final MetadataObject metadataObject; + private final MetadataObject newMetadataObject; + + private RenameMetadataObject(MetadataObject metadataObject, MetadataObject newMetadataObject) { + Preconditions.checkArgument( + !metadataObject.fullName().equals(newMetadataObject.fullName()), + "The metadata object must be different!"); + Preconditions.checkArgument( + metadataObject.type().equals(newMetadataObject.type()), + "The metadata object type must be same!"); + + this.metadataObject = metadataObject; + this.newMetadataObject = newMetadataObject; + } + + /** + * Returns the metadataObject to be renamed. + * + * @return return a metadataObject. + */ + public MetadataObject metadataObject() { + return metadataObject; + } + + /** + * Returns the new metadataObject object. + * + * @return return a metadataObject object. + */ + public MetadataObject newMetadataObject() { + return newMetadataObject; + } + + /** + * Compares this RenameMetadataObject instance with another object for equality. The comparison + * is based on the old metadata entity and new metadata entity. + * + * @param o The object to compare with this instance. + * @return true if the given object represents the same rename metadata entity; false otherwise. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RenameMetadataObject that = (RenameMetadataObject) o; + return metadataObject.equals(that.metadataObject) + && newMetadataObject.equals(that.newMetadataObject); + } + + /** + * Generates a hash code for this RenameMetadataObject instance. The hash code is based on the + * old metadata entity and new metadata entity. + * + * @return A hash code value for this update metadata entity operation. + */ + @Override + public int hashCode() { + return Objects.hash(metadataObject, newMetadataObject); + } + + /** + * Returns a string representation of the RenameMetadataObject instance. This string format + * includes the class name followed by the update metadata entity object operation. + * + * @return A string representation of the RenameMetadataObject instance. + */ + @Override + public String toString() { + return "RENAMEMETADATAOBJECT " + metadataObject + " " + newMetadataObject; + } + } + + /** A RemoveMetadataObject is to remove securable object's metadata entity. */ + final class RemoveMetadataObject implements MetadataObjectChange { + private final MetadataObject metadataObject; + + private RemoveMetadataObject(MetadataObject metadataObject) { + this.metadataObject = metadataObject; + } + + /** + * Returns the metadataObject to be renamed. + * + * @return return a metadataObject. + */ + public MetadataObject metadataObject() { + return metadataObject; + } + + /** + * Compares this RemoveMetadataObject instance with another object for equality. The comparison + * is based on the old metadata entity. + * + * @param o The object to compare with this instance. + * @return true if the given object represents the same rename metadata entity; false otherwise. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RenameMetadataObject that = (RenameMetadataObject) o; + return metadataObject.equals(that.metadataObject); + } + + /** + * Generates a hash code for this RemoveMetadataObject instance. The hash code is based on the + * old metadata entity. + * + * @return A hash code value for this update metadata entity operation. + */ + @Override + public int hashCode() { + return Objects.hash(metadataObject); + } + + /** + * Returns a string representation of the RemoveMetadataObject instance. This string format + * includes the class name followed by the remove metadata entity object operation. + * + * @return A string representation of the RemoveMetadataObject instance. + */ + @Override + public String toString() { + return "REMOVEMETADATAOBJECT " + metadataObject; + } + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java index c88f57f8e17..12072d7f688 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java @@ -138,7 +138,23 @@ public Set allowPrivilegesRule() { Privilege.Name.SELECT_TABLE); } + /** + * Allow Gravitino MetadataObject type defines rule. + * + * @return The allow Gravitino MetadataObject type defines rule. + */ + @Override + public Set allowMetadataObjectTypesRule() { + return ImmutableSet.of( + MetadataObject.Type.METALAKE, + MetadataObject.Type.CATALOG, + MetadataObject.Type.SCHEMA, + MetadataObject.Type.TABLE, + MetadataObject.Type.COLUMN); + } + /** Translate the Gravitino securable object to the Ranger owner securable object. */ + @Override public List translateOwner(MetadataObject gravitinoMetadataObject) { List rangerSecurableObjects = new ArrayList<>(); @@ -195,14 +211,14 @@ public List translateOwner(MetadataObject gravitinoMetada // Add `{schema}.{table}` for the TABLE permission rangerSecurableObjects.add( generateRangerSecurableObject( - convertToRangerMetadataObject(gravitinoMetadataObject), + translateMetadataObject(gravitinoMetadataObject).names(), RangerMetadataObject.Type.TABLE, ownerMappingRule())); // Add `{schema}.{table}.*` for the COLUMN permission rangerSecurableObjects.add( generateRangerSecurableObject( Stream.concat( - convertToRangerMetadataObject(gravitinoMetadataObject).stream(), + translateMetadataObject(gravitinoMetadataObject).names().stream(), Stream.of(RangerHelper.RESOURCE_ALL)) .collect(Collectors.toList()), RangerMetadataObject.Type.COLUMN, @@ -218,6 +234,7 @@ public List translateOwner(MetadataObject gravitinoMetada } /** Translate the Gravitino securable object to the Ranger securable object. */ + @Override public List translatePrivilege(SecurableObject securableObject) { List rangerSecurableObjects = new ArrayList<>(); @@ -352,14 +369,14 @@ public List translatePrivilege(SecurableObject securableO // Add `{schema}.{table}` for the TABLE permission rangerSecurableObjects.add( generateRangerSecurableObject( - convertToRangerMetadataObject(securableObject), + translateMetadataObject(securableObject).names(), RangerMetadataObject.Type.TABLE, rangerPrivileges)); // Add `{schema}.{table}.*` for the COLUMN permission rangerSecurableObjects.add( generateRangerSecurableObject( Stream.concat( - convertToRangerMetadataObject(securableObject).stream(), + translateMetadataObject(securableObject).names().stream(), Stream.of(RangerHelper.RESOURCE_ALL)) .collect(Collectors.toList()), RangerMetadataObject.Type.COLUMN, @@ -385,16 +402,39 @@ public List translatePrivilege(SecurableObject securableO } /** - * Because the Ranger securable object is different from the Gravitino securable object, we need - * to convert the Gravitino securable object to the Ranger securable object. + * Because the Ranger metadata object is different from the Gravitino metadata object, we need to + * convert the Gravitino metadata object to the Ranger metadata object. */ - List convertToRangerMetadataObject(MetadataObject metadataObject) { + @Override + public RangerMetadataObject translateMetadataObject(MetadataObject metadataObject) { + Preconditions.checkArgument( + allowMetadataObjectTypesRule().contains(metadataObject.type()), + String.format( + "The metadata object type %s is not supported in the RangerAuthorizationHivePlugin", + metadataObject.type())); Preconditions.checkArgument( !(metadataObject instanceof RangerPrivileges), "The metadata object must be not a RangerPrivileges object."); List nsMetadataObject = Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); - nsMetadataObject.remove(0); // remove the catalog name - return nsMetadataObject; + Preconditions.checkArgument( + nsMetadataObject.size() > 0, "The metadata object must have at least one name."); + + RangerMetadataObject.Type type; + if (metadataObject.type() == MetadataObject.Type.METALAKE + || metadataObject.type() == MetadataObject.Type.CATALOG) { + nsMetadataObject.clear(); + nsMetadataObject.add(RangerHelper.RESOURCE_ALL); + type = RangerMetadataObject.Type.SCHEMA; + } else { + nsMetadataObject.remove(0); // Remove the catalog name + type = RangerMetadataObject.Type.fromMetadataType(metadataObject.type()); + } + + validateRangerMetadataObject(nsMetadataObject, type); + return new RangerMetadataObjects.RangerMetadataObjectImpl( + RangerMetadataObjects.getParentFullName(nsMetadataObject), + RangerMetadataObjects.getLastName(nsMetadataObject), + type); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 965c7df3157..ff26d1ca67d 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -18,20 +18,29 @@ */ package org.apache.gravitino.authorization.ranger; +import static org.apache.gravitino.authorization.ranger.RangerMetadataObjects.DOT_JOINER; +import static org.apache.gravitino.authorization.ranger.RangerMetadataObjects.DOT_SPLITTER; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.MetadataObjectChange; import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Role; @@ -222,6 +231,38 @@ public Boolean onRoleUpdated(Role role, RoleChange... changes) throws RuntimeExc return Boolean.TRUE; } + @Override + public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws RuntimeException { + for (MetadataObjectChange change : changes) { + if (change instanceof MetadataObjectChange.RenameMetadataObject) { + MetadataObject metadataObject = + ((MetadataObjectChange.RenameMetadataObject) change).metadataObject(); + MetadataObject newMetadataObject = + ((MetadataObjectChange.RenameMetadataObject) change).newMetadataObject(); + RangerMetadataObject rangerMetadataObject = translateMetadataObject(metadataObject); + RangerMetadataObject newRangerMetadataObject = translateMetadataObject(newMetadataObject); + if (rangerMetadataObject.equals(newRangerMetadataObject)) { + LOG.info( + "The metadata object({}) and new metadata object({}) are equal, so ignore rename!", + rangerMetadataObject.fullName(), + newRangerMetadataObject.fullName()); + continue; + } + doRenameMetadataObject(rangerMetadataObject, newRangerMetadataObject); + } else if (change instanceof MetadataObjectChange.RemoveMetadataObject) { + MetadataObject metadataObject = + ((MetadataObjectChange.RemoveMetadataObject) change).metadataObject(); + RangerMetadataObject rangerMetadataObject = translateMetadataObject(metadataObject); + doRemoveMetadataObject(rangerMetadataObject); + } else { + throw new IllegalArgumentException( + "Unsupported metadata object change type: " + + (change == null ? "null" : change.getClass().getSimpleName())); + } + } + return Boolean.TRUE; + } + /** * Set or transfer the ownership of the metadata object.
* 1. If the metadata object already has an owner, then transfer the ownership from preOwner to @@ -704,20 +745,314 @@ private void removePolicyItemIfEqualRoleName( } } + /** + * IF remove the SCHEMA, need to remove these the relevant policies, `{schema}`, `{schema}.*`, + * `{schema}.*.*`
+ * IF remove the TABLE, need to remove these the relevant policies, `{schema}.*`, `{schema}.*.*` + *
+ * IF remove the COLUMN, Only need to remove `{schema}.*.*`
+ */ + private void doRemoveMetadataObject(RangerMetadataObject rangerMetadataObject) { + switch (rangerMetadataObject.type()) { + case SCHEMA: + doRemoveSchemaMetadataObject(rangerMetadataObject); + break; + case TABLE: + doRemoveTableMetadataObject(rangerMetadataObject); + break; + case COLUMN: + removePolicyByMetadataObject(rangerMetadataObject.names()); + break; + default: + throw new IllegalArgumentException( + "Unsupported metadata object type: " + rangerMetadataObject.type()); + } + } + + /** + * Remove the SCHEMA, Need to remove these the relevant policies, `{schema}`, `{schema}.*`, + * `{schema}.*.*` permissions. + */ + private void doRemoveSchemaMetadataObject(RangerMetadataObject rangerMetadataObject) { + Preconditions.checkArgument( + rangerMetadataObject.type() == RangerMetadataObject.Type.SCHEMA, + "The metadata object type must be SCHEMA"); + Preconditions.checkArgument( + rangerMetadataObject.names().size() == 1, "The metadata object names must be 1"); + if (RangerHelper.RESOURCE_ALL.equals(rangerMetadataObject.name())) { + // Delete metalake or catalog policies in this Ranger service + try { + List policies = rangerClient.getPoliciesInService(rangerServiceName); + policies.stream() + .forEach( + policy -> { + try { + rangerClient.deletePolicy(policy.getId()); + } catch (RangerServiceException e) { + LOG.error("Failed to rename the policy {}!", policy); + throw new RuntimeException(e); + } + }); + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + } else { + List> loop = + ImmutableList.of( + ImmutableList.of(rangerMetadataObject.name()) + /** SCHEMA permission */ + , + ImmutableList.of(rangerMetadataObject.name(), RangerHelper.RESOURCE_ALL) + /** TABLE permission */ + , + ImmutableList.of( + rangerMetadataObject.name(), RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL) + /** COLUMN permission */ + ); + for (List resNames : loop) { + removePolicyByMetadataObject(resNames); + } + } + } + + /** + * Remove the TABLE, Need to remove these the relevant policies, `*.{table}`, `*.{table}.{column}` + * permissions. + */ + private void doRemoveTableMetadataObject(RangerMetadataObject rangerMetadataObject) { + List> loop = + ImmutableList.of( + rangerMetadataObject.names() + /** TABLE permission */ + , + Stream.concat( + rangerMetadataObject.names().stream(), Stream.of(RangerHelper.RESOURCE_ALL)) + .collect(Collectors.toList()) + /** COLUMN permission */ + ); + for (List resNames : loop) { + removePolicyByMetadataObject(resNames); + } + } + + /** + * IF rename the SCHEMA, Need to rename these the relevant policies, `{schema}`, `{schema}.*`, + * `{schema}.*.*`
+ * IF rename the TABLE, Need to rename these the relevant policies, `{schema}.*`, `{schema}.*.*` + *
+ * IF rename the COLUMN, Only need to rename `{schema}.*.*`
+ */ + private void doRenameMetadataObject( + RangerMetadataObject rangerMetadataObject, RangerMetadataObject newRangerMetadataObject) { + switch (rangerMetadataObject.type()) { + case SCHEMA: + doRenameSchemaMetadataObject(rangerMetadataObject, newRangerMetadataObject); + break; + case TABLE: + doRenameTableMetadataObject(rangerMetadataObject, newRangerMetadataObject); + break; + case COLUMN: + doRenameColumnMetadataObject(rangerMetadataObject, newRangerMetadataObject); + break; + default: + throw new IllegalArgumentException( + "Unsupported metadata object type: " + rangerMetadataObject.type()); + } + } + + /** + * Rename the SCHEMA, Need to rename these the relevant policies, `{schema}`, `{schema}.*`, + * `{schema}.*.*`
+ */ + private void doRenameSchemaMetadataObject( + RangerMetadataObject rangerMetadataObject, RangerMetadataObject newRangerMetadataObject) { + List oldMetadataNames = new ArrayList<>(); + List newMetadataNames = new ArrayList<>(); + List> loop = + ImmutableList.of( + ImmutableMap.of( + rangerMetadataObject.names().get(0), newRangerMetadataObject.names().get(0)), + ImmutableMap.of(RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), + ImmutableMap.of(RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL)); + for (Map mapName : loop) { + oldMetadataNames.add(mapName.keySet().stream().findFirst().get()); + newMetadataNames.add(mapName.values().stream().findFirst().get()); + updatePolicyByMetadataObject(MetadataObject.Type.SCHEMA, oldMetadataNames, newMetadataNames); + } + } + + /** + * Rename the TABLE, Need to rename these the relevant policies, `*.{table}`, `*.{table}.{column}` + *
+ */ + private void doRenameTableMetadataObject( + RangerMetadataObject rangerMetadataObject, RangerMetadataObject newRangerMetadataObject) { + List oldMetadataNames = new ArrayList<>(); + List newMetadataNames = new ArrayList<>(); + List> loop = + ImmutableList.of( + ImmutableMap.of(rangerMetadataObject.names().get(0), MetadataObject.Type.SCHEMA), + ImmutableMap.of(rangerMetadataObject.names().get(1), MetadataObject.Type.TABLE), + ImmutableMap.of(RangerHelper.RESOURCE_ALL, MetadataObject.Type.COLUMN)); + for (Map nameAndType : loop) { + oldMetadataNames.add(nameAndType.keySet().stream().findFirst().get()); + if (nameAndType.containsValue(MetadataObject.Type.SCHEMA)) { + newMetadataNames.add(newRangerMetadataObject.names().get(0)); + // Skip update the schema name operation + continue; + } else if (nameAndType.containsValue(MetadataObject.Type.TABLE)) { + newMetadataNames.add(newRangerMetadataObject.names().get(1)); + } else if (nameAndType.containsValue(MetadataObject.Type.COLUMN)) { + newMetadataNames.add(RangerHelper.RESOURCE_ALL); + } + updatePolicyByMetadataObject(MetadataObject.Type.TABLE, oldMetadataNames, newMetadataNames); + } + } + + /** rename the COLUMN, Only need to rename `*.*.{column}`
*/ + private void doRenameColumnMetadataObject( + RangerMetadataObject rangerMetadataObject, RangerMetadataObject newRangerMetadataObject) { + List oldMetadataNames = new ArrayList<>(); + List newMetadataNames = new ArrayList<>(); + List> loop = + ImmutableList.of( + ImmutableMap.of(rangerMetadataObject.names().get(0), MetadataObject.Type.SCHEMA), + ImmutableMap.of(rangerMetadataObject.names().get(1), MetadataObject.Type.TABLE), + ImmutableMap.of(rangerMetadataObject.names().get(2), MetadataObject.Type.COLUMN)); + for (Map nameAndType : loop) { + oldMetadataNames.add(nameAndType.keySet().stream().findFirst().get()); + if (nameAndType.containsValue(MetadataObject.Type.SCHEMA)) { + newMetadataNames.add(newRangerMetadataObject.names().get(0)); + // Skip update the schema name operation + continue; + } else if (nameAndType.containsValue(MetadataObject.Type.TABLE)) { + newMetadataNames.add(newRangerMetadataObject.names().get(1)); + // Skip update the table name operation + continue; + } else if (nameAndType.containsValue(MetadataObject.Type.COLUMN)) { + newMetadataNames.add(newRangerMetadataObject.names().get(2)); + } + updatePolicyByMetadataObject(MetadataObject.Type.COLUMN, oldMetadataNames, newMetadataNames); + } + } + + /** + * Remove the policy by the metadata object names.
+ * + * @param metadataNames The metadata object names. + */ + private void removePolicyByMetadataObject(List metadataNames) { + List policies = rangerHelper.wildcardSearchPolies(metadataNames); + Map preciseFilters = new HashMap<>(); + for (int i = 0; i < metadataNames.size(); i++) { + preciseFilters.put(rangerHelper.policyResourceDefines.get(i), metadataNames.get(i)); + } + policies = + policies.stream() + .filter( + policy -> + policy.getResources().entrySet().stream() + .allMatch( + entry -> + preciseFilters.containsKey(entry.getKey()) + && entry.getValue().getValues().size() == 1 + && entry + .getValue() + .getValues() + .contains(preciseFilters.get(entry.getKey())))) + .collect(Collectors.toList()); + policies.stream() + .forEach( + policy -> { + try { + rangerClient.deletePolicy(policy.getId()); + } catch (RangerServiceException e) { + LOG.error("Failed to rename the policy {}!", policy); + throw new RuntimeException(e); + } + }); + } + + private void updatePolicyByMetadataObject( + MetadataObject.Type operationType, + List oldMetadataNames, + List newMetadataNames) { + List oldPolicies = rangerHelper.wildcardSearchPolies(oldMetadataNames); + List existNewPolicies = rangerHelper.wildcardSearchPolies(newMetadataNames); + if (oldPolicies.isEmpty()) { + LOG.warn("Cannot find the Ranger policy for the metadata object({})!", oldMetadataNames); + } + if (!existNewPolicies.isEmpty()) { + LOG.warn("The Ranger policy for the metadata object({}) already exists!", newMetadataNames); + } + Map operationTypeIndex = + ImmutableMap.of( + MetadataObject.Type.SCHEMA, 0, + MetadataObject.Type.TABLE, 1, + MetadataObject.Type.COLUMN, 2); + oldPolicies.stream() + .forEach( + policy -> { + try { + // Update the policy name + String policyName = policy.getName(); + List policyNames = Lists.newArrayList(DOT_SPLITTER.splitToList(policyName)); + Preconditions.checkArgument( + policyNames.size() >= oldMetadataNames.size(), + String.format("The policy name(%s) is invalid!", policyName)); + int index = operationTypeIndex.get(operationType); + if (policyNames.get(index).equals(RangerHelper.RESOURCE_ALL)) { + // Doesn't need to rename the policy `*` + return; + } + policyNames.set(index, newMetadataNames.get(index)); + // Update the policy resource name to new name + policy + .getResources() + .put( + rangerHelper.policyResourceDefines.get(index), + new RangerPolicy.RangerPolicyResource(newMetadataNames.get(index))); + policy.setName(DOT_JOINER.join(policyNames)); + + boolean alreadyExist = + existNewPolicies.stream() + .anyMatch( + existNewPolicy -> + existNewPolicy.getName().equals(policy.getName()) + || existNewPolicy.getResources().equals(policy.getResources())); + if (alreadyExist) { + LOG.warn( + "The Ranger policy for the metadata object({}) already exists!", + newMetadataNames); + return; + } + + // Update the policy + rangerClient.updatePolicy(policy.getId(), policy); + } catch (RangerServiceException e) { + LOG.error("Failed to rename the policy {}!", policy); + throw new RuntimeException(e); + } + }); + } + @Override public void close() throws IOException {} - /** Generate different Ranger securable object */ + /** Generate Ranger securable object */ public RangerSecurableObject generateRangerSecurableObject( List names, RangerMetadataObject.Type type, Set privileges) { validateRangerMetadataObject(names, type); - RangerMetadataObject metadataObject = + RangerMetadataObject rangerMetadataObject = new RangerMetadataObjects.RangerMetadataObjectImpl( RangerMetadataObjects.getParentFullName(names), RangerMetadataObjects.getLastName(names), type); return new RangerSecurableObjects.RangerSecurableObjectImpl( - metadataObject.parent(), metadataObject.name(), metadataObject.type(), privileges); + rangerMetadataObject.parent(), + rangerMetadataObject.name(), + rangerMetadataObject.type(), + privileges); } public boolean validAuthorizationOperation(List securableObjects) { diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java index 51a61560fa0..86ed2ee88a7 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java @@ -53,7 +53,7 @@ public class RangerHelper { /** The owner privileges, the owner can do anything on the metadata object */ private final Set ownerPrivileges; /** The policy search keys */ - private final List policyResourceDefines; + protected final List policyResourceDefines; private final RangerClient rangerClient; private final String rangerAdminName; @@ -163,72 +163,83 @@ void addPolicyItem(RangerPolicy policy, String roleName, RangerSecurableObject s } /** - * Find the managed policy for the ranger securable object. + * Find the managed policies for the ranger securable object. * - * @param rangerMetadataObject The ranger securable object to find the managed policy. + * @param metadataNames The metadata object names to find the managed policy. * @return The managed policy for the metadata object. */ - public RangerPolicy findManagedPolicy(RangerMetadataObject rangerMetadataObject) + public List wildcardSearchPolies(List metadataNames) throws AuthorizationPluginException { - List nsMetadataObj = rangerMetadataObject.names(); - Map searchFilters = new HashMap<>(); - Map preciseFilters = new HashMap<>(); searchFilters.put(SearchFilter.SERVICE_NAME, rangerServiceName); searchFilters.put(SearchFilter.POLICY_LABELS_PARTIAL, MANAGED_BY_GRAVITINO); - for (int i = 0; i < nsMetadataObj.size(); i++) { + for (int i = 0; i < metadataNames.size(); i++) { searchFilters.put( - SearchFilter.RESOURCE_PREFIX + policyResourceDefines.get(i), nsMetadataObj.get(i)); - preciseFilters.put(policyResourceDefines.get(i), nsMetadataObj.get(i)); + SearchFilter.RESOURCE_PREFIX + policyResourceDefines.get(i), metadataNames.get(i)); } try { List policies = rangerClient.findPolicies(searchFilters); + return policies; + } catch (RangerServiceException e) { + throw new AuthorizationPluginException(e); + } + } - if (!policies.isEmpty()) { - /** - * Because Ranger doesn't support the precise search, Ranger will return the policy meets - * the wildcard(*,?) conditions, If you use `db.table` condition to search policy, the - * Ranger will match `db1.table1`, `db1.table2`, `db*.table*`, So we need to manually - * precisely filter this research results. - */ - policies = - policies.stream() - .filter( - policy -> - policy.getResources().entrySet().stream() - .allMatch( - entry -> - preciseFilters.containsKey(entry.getKey()) - && entry.getValue().getValues().size() == 1 - && entry - .getValue() - .getValues() - .contains(preciseFilters.get(entry.getKey())))) - .collect(Collectors.toList()); - } - - // Only return the policies that are managed by Gravitino. - if (policies.size() > 1) { - throw new AuthorizationPluginException( - "Every metadata object has only a Gravitino managed policy."); + /** + * Find the managed policy for the ranger securable object. + * + * @param rangerMetadataObject The ranger securable object to find the managed policy. + * @return The managed policy for the metadata object. + */ + public RangerPolicy findManagedPolicy(RangerMetadataObject rangerMetadataObject) + throws AuthorizationPluginException { + List policies = wildcardSearchPolies(rangerMetadataObject.names()); + if (!policies.isEmpty()) { + /** + * Because Ranger doesn't support the precise search, Ranger will return the policy meets the + * wildcard(*,?) conditions, If you use `db.table` condition to search policy, the Ranger will + * match `db1.table1`, `db1.table2`, `db*.table*`, So we need to manually precisely filter + * this research results. + */ + List nsMetadataObj = rangerMetadataObject.names(); + Map preciseFilters = new HashMap<>(); + for (int i = 0; i < nsMetadataObj.size(); i++) { + preciseFilters.put(policyResourceDefines.get(i), nsMetadataObj.get(i)); } + policies = + policies.stream() + .filter( + policy -> + policy.getResources().entrySet().stream() + .allMatch( + entry -> + preciseFilters.containsKey(entry.getKey()) + && entry.getValue().getValues().size() == 1 + && entry + .getValue() + .getValues() + .contains(preciseFilters.get(entry.getKey())))) + .collect(Collectors.toList()); + } + // Only return the policies that are managed by Gravitino. + if (policies.size() > 1) { + throw new AuthorizationPluginException( + "Every metadata object has only a Gravitino managed policy."); + } - if (policies.isEmpty()) { - return null; - } + if (policies.isEmpty()) { + return null; + } - RangerPolicy policy = policies.get(0); - // Delegating Gravitino management policies cannot contain duplicate privilege - policy.getPolicyItems().forEach(this::checkPolicyItemAccess); - policy.getDenyPolicyItems().forEach(this::checkPolicyItemAccess); - policy.getRowFilterPolicyItems().forEach(this::checkPolicyItemAccess); - policy.getDataMaskPolicyItems().forEach(this::checkPolicyItemAccess); + RangerPolicy policy = policies.get(0); + // Delegating Gravitino management policies cannot contain duplicate privilege + policy.getPolicyItems().forEach(this::checkPolicyItemAccess); + policy.getDenyPolicyItems().forEach(this::checkPolicyItemAccess); + policy.getRowFilterPolicyItems().forEach(this::checkPolicyItemAccess); + policy.getDataMaskPolicyItems().forEach(this::checkPolicyItemAccess); - return policy; - } catch (RangerServiceException e) { - throw new AuthorizationPluginException(e); - } + return policy; } protected boolean checkRangerRole(String roleName) throws AuthorizationPluginException { @@ -373,9 +384,7 @@ protected RangerPolicy createPolicyAddResources(RangerMetadataObject metadataObj policy.setService(rangerServiceName); policy.setName(metadataObject.fullName()); policy.setPolicyLabels(Lists.newArrayList(RangerHelper.MANAGED_BY_GRAVITINO)); - List nsMetadataObject = metadataObject.names(); - for (int i = 0; i < nsMetadataObject.size(); i++) { RangerPolicy.RangerPolicyResource policyResource = new RangerPolicy.RangerPolicyResource(nsMetadataObject.get(i)); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObjects.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObjects.java index de2002df14c..7c7bed69d27 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObjects.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerMetadataObjects.java @@ -21,15 +21,14 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; -import com.google.common.collect.Lists; import java.util.List; import org.apache.gravitino.MetadataObject; /** The helper class for {@link RangerMetadataObject}. */ public class RangerMetadataObjects { - private static final Splitter DOT_SPLITTER = Splitter.on('.'); + protected static final Splitter DOT_SPLITTER = Splitter.on('.'); - private static final Joiner DOT_JOINER = Joiner.on('.'); + protected static final Joiner DOT_JOINER = Joiner.on('.'); private RangerMetadataObjects() {} @@ -84,7 +83,7 @@ public String name() { @Override public List names() { - return Lists.newArrayList(DOT_SPLITTER.splitToList(fullName())); + return DOT_SPLITTER.splitToList(fullName()); } @Override diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java index 6b30b4bb1c6..ca11aaf704a 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java @@ -59,6 +59,13 @@ public interface RangerPrivilegesMappingProvider { */ Set allowPrivilegesRule(); + /** + * Allow Gravitino MetadataObject type defines rule. + * + * @return The allow Gravitino MetadataObject type defines rule. + */ + Set allowMetadataObjectTypesRule(); + /** * Translate the Gravitino securable object to the Ranger securable object. * @@ -74,4 +81,12 @@ public interface RangerPrivilegesMappingProvider { * @return The Ranger owner securable object list. */ List translateOwner(MetadataObject metadataObject); + + /** + * Translate the Gravitino metadata object to the Ranger metadata object. + * + * @param metadataObject The Gravitino metadata object. + * @return The Ranger metadata object. + */ + RangerMetadataObject translateMetadataObject(MetadataObject metadataObject); } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java index 21ab5dc55b1..97f2b903527 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationPluginIT.java @@ -29,6 +29,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; +import org.apache.gravitino.authorization.ranger.RangerHelper; import org.apache.gravitino.authorization.ranger.RangerMetadataObject; import org.apache.gravitino.authorization.ranger.RangerSecurableObject; import org.junit.jupiter.api.Assertions; @@ -46,6 +47,38 @@ public static void setup() { rangerAuthPlugin = RangerITEnv.rangerAuthHivePlugin; } + @Test + public void testTranslateMetadataObject() { + MetadataObject metalake = + MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); + RangerMetadataObject rangerMetalake = rangerAuthPlugin.translateMetadataObject(metalake); + Assertions.assertEquals(1, rangerMetalake.names().size()); + Assertions.assertEquals(RangerHelper.RESOURCE_ALL, rangerMetalake.names().get(0)); + Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, rangerMetalake.type()); + + MetadataObject catalog = + MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); + RangerMetadataObject rangerCatalog = rangerAuthPlugin.translateMetadataObject(catalog); + Assertions.assertEquals(1, rangerCatalog.names().size()); + Assertions.assertEquals(RangerHelper.RESOURCE_ALL, rangerCatalog.names().get(0)); + Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, rangerCatalog.type()); + + MetadataObject schema = + MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); + RangerMetadataObject rangerSchema = rangerAuthPlugin.translateMetadataObject(schema); + Assertions.assertEquals(1, rangerSchema.names().size()); + Assertions.assertEquals("schema1", rangerSchema.names().get(0)); + Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, rangerSchema.type()); + + MetadataObject table = + MetadataObjects.parse(String.format("catalog1.schema1.tab1"), MetadataObject.Type.TABLE); + RangerMetadataObject rangerTable = rangerAuthPlugin.translateMetadataObject(table); + Assertions.assertEquals(2, rangerTable.names().size()); + Assertions.assertEquals("schema1", rangerTable.names().get(0)); + Assertions.assertEquals("tab1", rangerTable.names().get(1)); + Assertions.assertEquals(RangerMetadataObject.Type.TABLE, rangerTable.type()); + } + @Test public void testTranslatePrivilege() { SecurableObject createSchemaInMetalake = @@ -56,7 +89,7 @@ public void testTranslatePrivilege() { List createSchemaInMetalake1 = rangerAuthPlugin.translatePrivilege(createSchemaInMetalake); Assertions.assertEquals(1, createSchemaInMetalake1.size()); - Assertions.assertEquals("*", createSchemaInMetalake1.get(0).fullName()); + Assertions.assertEquals(RangerHelper.RESOURCE_ALL, createSchemaInMetalake1.get(0).fullName()); Assertions.assertEquals( RangerMetadataObject.Type.SCHEMA, createSchemaInMetalake1.get(0).type()); @@ -68,7 +101,7 @@ public void testTranslatePrivilege() { List createSchemaInCatalog1 = rangerAuthPlugin.translatePrivilege(createSchemaInCatalog); Assertions.assertEquals(1, createSchemaInCatalog1.size()); - Assertions.assertEquals("*", createSchemaInCatalog1.get(0).fullName()); + Assertions.assertEquals(RangerHelper.RESOURCE_ALL, createSchemaInCatalog1.get(0).fullName()); Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, createSchemaInCatalog1.get(0).type()); for (Privilege privilege : @@ -136,7 +169,7 @@ public void testTranslateOwner() { MetadataObject metalake = MetadataObjects.parse("metalake_or_catalog", type); List metalakeOwner = rangerAuthPlugin.translateOwner(metalake); Assertions.assertEquals(3, metalakeOwner.size()); - Assertions.assertEquals("*", metalakeOwner.get(0).fullName()); + Assertions.assertEquals(RangerHelper.RESOURCE_ALL, metalakeOwner.get(0).fullName()); Assertions.assertEquals(RangerMetadataObject.Type.SCHEMA, metalakeOwner.get(0).type()); Assertions.assertEquals("*.*", metalakeOwner.get(1).fullName()); Assertions.assertEquals(RangerMetadataObject.Type.TABLE, metalakeOwner.get(1).type()); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index 91d58bd1873..565acb82f4a 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -201,7 +201,6 @@ private static void generateRangerSparkSecurityXML() throws IOException { @AfterAll public void stop() { - RangerITEnv.cleanup(); if (client != null) { Arrays.stream(catalog.asSchemas().listSchemas()) .filter(schema -> !schema.equals("default")) @@ -222,8 +221,8 @@ public void stop() { } catch (Exception e) { LOG.error("Failed to close CloseableGroup", e); } - client = null; + RangerITEnv.cleanup(); } @Test diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index 85aa2f5b2a8..00a231e8068 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -19,6 +19,7 @@ package org.apache.gravitino.authorization.ranger.integration.test; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; +import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.rangerClient; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.verifyRoleInRanger; import com.google.common.collect.ImmutableList; @@ -34,6 +35,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.authorization.MetadataObjectChange; import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Privileges; @@ -87,26 +89,7 @@ public static void stop() { @AfterEach public void clean() { - // Clean up the test Ranger policy - Role mockCatalogRole = mockCatalogRole(currentFunName()); - mockCatalogRole.securableObjects().stream() - .forEach( - securableObject -> { - rangerAuthHivePlugin.translatePrivilege(securableObject).stream() - .forEach( - rangerSecurableObject -> { - deleteHivePolicy(rangerSecurableObject); - }); - }); - mockCatalogRole.securableObjects().stream() - .forEach( - securableObject -> { - rangerAuthHivePlugin.translateOwner(securableObject).stream() - .forEach( - rangerSecurableObject -> { - deleteHivePolicy(rangerSecurableObject); - }); - }); + RangerITEnv.cleanAllPolicy(RangerITEnv.RANGER_HIVE_REPO_NAME); } /** @@ -613,6 +596,326 @@ public void testRoleChangeUpdateCatalogSecurableObject() { assertFindManagedPolicy(mockCatalogRole, false); } + @Test + public void testMetadataObjectChangeRenameMetalake() { + String currentFunName = currentFunName(); + MetadataObject oldMetadataObject = + MetadataObjects.parse( + String.format("metalake-%s", currentFunName), MetadataObject.Type.METALAKE); + SecurableObject oldSecurableObject1 = + SecurableObjects.parse( + oldMetadataObject.fullName(), + oldMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(oldSecurableObject1)) + .build(); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); + + MetadataObject newMetadataObject = + MetadataObjects.parse( + String.format("metalake-new-%s", currentFunName), oldMetadataObject.type()); + Assertions.assertTrue( + rangerAuthHivePlugin.onMetadataUpdated( + MetadataObjectChange.rename(oldMetadataObject, newMetadataObject))); + SecurableObject newSecurableObject1 = + SecurableObjects.parse( + newMetadataObject.fullName(), + newMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity newRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(newSecurableObject1)) + .build(); + assertFindManagedPolicy(newRole, true); + } + + @Test + public void testMetadataObjectChangeRenameCatalog() { + String currentFunName = currentFunName(); + MetadataObject oldMetadataObject = + MetadataObjects.parse( + String.format("catalog-%s", currentFunName), MetadataObject.Type.CATALOG); + SecurableObject oldSecurableObject1 = + SecurableObjects.parse( + oldMetadataObject.fullName(), + oldMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(oldSecurableObject1)) + .build(); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); + + MetadataObject newMetadataObject = + MetadataObjects.parse( + String.format("catalog-new-%s", currentFunName), oldMetadataObject.type()); + Assertions.assertTrue( + rangerAuthHivePlugin.onMetadataUpdated( + MetadataObjectChange.rename(oldMetadataObject, newMetadataObject))); + SecurableObject newSecurableObject1 = + SecurableObjects.parse( + newMetadataObject.fullName(), + newMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity newRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(newSecurableObject1)) + .build(); + assertFindManagedPolicy(newRole, true); + } + + @Test + public void testMetadataObjectChangeRenameSchema() { + String currentFunName = currentFunName(); + MetadataObject oldMetadataObject = + MetadataObjects.parse( + String.format("catalog.old-%s", currentFunName), MetadataObject.Type.SCHEMA); + SecurableObject oldSecurableObject1 = + SecurableObjects.parse( + oldMetadataObject.fullName(), + oldMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(oldSecurableObject1)) + .build(); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); + + MetadataObject newMetadataObject = + MetadataObjects.parse( + String.format("catalog.new-%s", currentFunName), oldMetadataObject.type()); + Assertions.assertTrue( + rangerAuthHivePlugin.onMetadataUpdated( + MetadataObjectChange.rename(oldMetadataObject, newMetadataObject))); + assertFindManagedPolicy(role, false); + SecurableObject newSecurableObject1 = + SecurableObjects.parse( + newMetadataObject.fullName(), + newMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity newRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(newSecurableObject1)) + .build(); + assertFindManagedPolicy(newRole, true); + } + + @Test + public void testMetadataObjectChangeRenameTable() { + String currentFunName = currentFunName(); + MetadataObject oldMetadataObject = + MetadataObjects.parse( + String.format("catalog.schema1.old-%s", currentFunName), MetadataObject.Type.TABLE); + SecurableObject oldSecurableObject1 = + SecurableObjects.parse( + oldMetadataObject.fullName(), + oldMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(oldSecurableObject1)) + .build(); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); + + MetadataObject newMetadataObject = + MetadataObjects.parse( + String.format("catalog.schema1.new-%s", currentFunName), oldMetadataObject.type()); + Assertions.assertTrue( + rangerAuthHivePlugin.onMetadataUpdated( + MetadataObjectChange.rename(oldMetadataObject, newMetadataObject))); + assertFindManagedPolicy(role, false); + SecurableObject newSecurableObject1 = + SecurableObjects.parse( + newMetadataObject.fullName(), + newMetadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity newRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(newSecurableObject1)) + .build(); + assertFindManagedPolicy(newRole, true); + } + + @Test + public void testMetadataObjectChangeRemoveMetalake() throws RangerServiceException { + metadataObjectChangeRemoveMetalakeOrCatalog(currentFunName(), MetadataObject.Type.METALAKE); + } + + @Test + public void testMetadataObjectChangeRemoveCatalog() throws RangerServiceException { + metadataObjectChangeRemoveMetalakeOrCatalog(currentFunName(), MetadataObject.Type.CATALOG); + } + + void metadataObjectChangeRemoveMetalakeOrCatalog(String funcName, MetadataObject.Type type) + throws RangerServiceException { + createHivePolicy( + Lists.newArrayList(String.format("%s*", funcName), "*"), + GravitinoITUtils.genRandomName(currentFunName())); + createHivePolicy( + Lists.newArrayList(String.format("%s*", funcName), "tab*"), + GravitinoITUtils.genRandomName(currentFunName())); + createHivePolicy( + Lists.newArrayList(String.format("%s3", funcName), "*"), + GravitinoITUtils.genRandomName(currentFunName())); + createHivePolicy( + Lists.newArrayList(String.format("%s3", funcName), "tab*"), + GravitinoITUtils.genRandomName(currentFunName())); + Assertions.assertEquals( + 4, rangerClient.getPoliciesInService(RangerITEnv.RANGER_HIVE_REPO_NAME).size()); + + String currentFunName = currentFunName(); + MetadataObject metadataObject = + MetadataObjects.parse(String.format("metalake-%s", currentFunName), type); + SecurableObject oldSecurableObject1 = + SecurableObjects.parse( + metadataObject.fullName(), + metadataObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(oldSecurableObject1)) + .build(); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); + + Assertions.assertTrue( + rangerAuthHivePlugin.onMetadataUpdated(MetadataObjectChange.remove(metadataObject))); + assertFindManagedPolicy(role, false); + + Assertions.assertEquals( + 0, rangerClient.getPoliciesInService(RangerITEnv.RANGER_HIVE_REPO_NAME).size()); + } + + @Test + public void testMetadataObjectChangeRemoveSchema() throws RangerServiceException { + String currentFunName = currentFunName(); + MetadataObject schemaObject = + MetadataObjects.parse( + String.format("catalog.old-%s", currentFunName), MetadataObject.Type.SCHEMA); + SecurableObject schemaSecurableObject = + SecurableObjects.parse( + schemaObject.fullName(), + schemaObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + MetadataObject tableObject = + MetadataObjects.parse( + String.format("catalog.old-%s.tab1", currentFunName), MetadataObject.Type.TABLE); + SecurableObject tableSecurableObject = + SecurableObjects.parse( + tableObject.fullName(), + tableObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(schemaSecurableObject, tableSecurableObject)) + .build(); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); + Assertions.assertEquals( + 4, rangerClient.getPoliciesInService(RangerITEnv.RANGER_HIVE_REPO_NAME).size()); + + Assertions.assertTrue( + rangerAuthHivePlugin.onMetadataUpdated(MetadataObjectChange.remove(schemaObject))); + RoleEntity roleVerify = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(schemaSecurableObject)) + .build(); + assertFindManagedPolicy(roleVerify, false); + Assertions.assertEquals( + 2, rangerClient.getPoliciesInService(RangerITEnv.RANGER_HIVE_REPO_NAME).size()); + } + + @Test + public void testMetadataObjectChangeRemoveTable() throws RangerServiceException { + String currentFunName = currentFunName(); + MetadataObject schemaObject = + MetadataObjects.parse( + String.format("catalog.old-%s", currentFunName), MetadataObject.Type.SCHEMA); + SecurableObject securableObjectSchema = + SecurableObjects.parse( + schemaObject.fullName(), + schemaObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + MetadataObject tableObject = + MetadataObjects.parse( + String.format("catalog.old-%s.tab1", currentFunName), MetadataObject.Type.TABLE); + SecurableObject securableObjectTab = + SecurableObjects.parse( + tableObject.fullName(), + tableObject.type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObjectSchema, securableObjectTab)) + .build(); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); + + Assertions.assertTrue( + rangerAuthHivePlugin.onMetadataUpdated(MetadataObjectChange.remove(tableObject))); + RoleEntity verifyScheamRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObjectSchema)) + .build(); + RoleEntity verifyTableRole = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(securableObjectTab)) + .build(); + assertFindManagedPolicy(verifyScheamRole, true); + assertFindManagedPolicy(verifyTableRole, false); + Assertions.assertEquals( + 2, rangerClient.getPoliciesInService(RangerITEnv.RANGER_HIVE_REPO_NAME).size()); + } + + @Test public void testRoleChangeCombinedOperation() { MetadataObject oldMetadataObject = MetadataObjects.parse( diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 653d1720204..be653bd3dbc 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.authorization.ranger.integration.test; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.util.Arrays; @@ -117,24 +118,17 @@ public static void init() { createRangerHdfsRepository("", true); createRangerHiveRepository("", true); allowAnyoneAccessHDFS(); - allowAnyoneAccessInformationSchema(); initRangerService = true; } } } public static void cleanup() { - try { - if (rangerClient != null) { - if (rangerClient.getService(RANGER_TRINO_REPO_NAME) != null) { - rangerClient.deleteService(RANGER_TRINO_REPO_NAME); - } - if (rangerClient.getService(RANGER_HIVE_REPO_NAME) != null) { - rangerClient.deleteService(RANGER_HIVE_REPO_NAME); - } + if (rangerClient != null) { + // Clean up the test Ranger policy + for (String repoName : ImmutableList.of(RANGER_HIVE_REPO_NAME, RANGER_HDFS_REPO_NAME)) { + cleanAllPolicy(repoName); } - } catch (RangerServiceException e) { - // ignore } } @@ -598,11 +592,17 @@ protected static void updateOrCreateRangerPolicy( /** Clean all policy in the Ranger */ protected static void cleanAllPolicy(String serviceName) { try { - List policies = - rangerClient.findPolicies(ImmutableMap.of(SearchFilter.SERVICE_NAME, serviceName)); - for (RangerPolicy policy : policies) { - rangerClient.deletePolicy(policy.getId()); - } + List policies = rangerClient.getPoliciesInService(serviceName); + policies.stream() + .forEach( + policy -> { + try { + rangerClient.deletePolicy(policy.getId()); + } catch (RangerServiceException e) { + LOG.error("Failed to rename the policy {}!", policy); + throw new RuntimeException(e); + } + }); } catch (RangerServiceException e) { throw new RuntimeException(e); } diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/AuthorizationPlugin.java b/core/src/main/java/org/apache/gravitino/connector/authorization/AuthorizationPlugin.java index 7c9559782c1..8ac75a649a5 100644 --- a/core/src/main/java/org/apache/gravitino/connector/authorization/AuthorizationPlugin.java +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/AuthorizationPlugin.java @@ -26,4 +26,7 @@ * permission system, the implementation method of these function interface must be idempotent. */ public interface AuthorizationPlugin - extends UserGroupAuthorizationPlugin, RoleAuthorizationPlugin, Closeable {} + extends UserGroupAuthorizationPlugin, + RoleAuthorizationPlugin, + MetadataAuthorizationPlugin, + Closeable {} diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/MetadataAuthorizationPlugin.java b/core/src/main/java/org/apache/gravitino/connector/authorization/MetadataAuthorizationPlugin.java new file mode 100644 index 00000000000..f228d98c74f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/MetadataAuthorizationPlugin.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.connector.authorization; + +import org.apache.gravitino.authorization.MetadataObjectChange; + +/** + * Interface for authorization User and Group plugin operation of the underlying access control + * system. + */ +interface MetadataAuthorizationPlugin { + /** + * After updating a metadata object in Gravitino, this method is called to update the role in the + * underlying system.
+ * + * @param changes metadata object changes apply to the role. + * @return True if the update operation is successful; False if the update operation fails. + * @throws RuntimeException If update role encounters storage issues. + */ + Boolean onMetadataUpdated(MetadataObjectChange... changes) throws RuntimeException; +} diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java index e28dffc32da..e078eda410e 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.MetadataObjectChange; import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.Role; import org.apache.gravitino.authorization.RoleChange; @@ -110,4 +111,9 @@ public Boolean onRevokedRolesFromGroup(List roles, Group group) throws Run @Override public void close() throws IOException {} + + @Override + public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws RuntimeException { + return null; + } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java index 1ed1ab9dc82..8a68f825d0e 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.MetadataObjectChange; import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.Role; import org.apache.gravitino.authorization.RoleChange; @@ -110,4 +111,9 @@ public Boolean onRevokedRolesFromGroup(List roles, Group group) throws Run @Override public void close() throws IOException {} + + @Override + public Boolean onMetadataUpdated(MetadataObjectChange... changes) throws RuntimeException { + return null; + } }