[#5775] feat(auth): Chain authorization plugin framework (#5786)

### What changes were proposed in this pull request? 1. Add Chain auth plugin module 1. Add auth common module 3. Add Chain authorization Ranger Hive and Ranger HDFS ITs ### Why are the changes needed? Fix: #5775 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Add ITs
apache · Dec 25, 2024 · 3e7e550 · 3e7e550
1 parent a58b7f8
commit 3e7e550
Show file tree

Hide file tree

Showing 49 changed files with 1,630 additions and 335 deletions.
diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml
@@ -87,12 +87,9 @@ jobs:
       - name: Authorization Integration Test (JDK${{ matrix.java-version }})
         id: integrationTest
         run: |
-          ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test
-          ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test
-          ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test
-          ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test
-          ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test
-          ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test
+          ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test
+          ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test
+          ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3

diff --git a/authorizations/authorization-chain/build.gradle.kts b/authorizations/authorization-chain/build.gradle.kts
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+description = "authorization-chain"
+
+plugins {
+  `maven-publish`
+  id("java")
+  id("idea")
+}
+
+val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
+val sparkVersion: String = libs.versions.spark35.get()
+val kyuubiVersion: String = libs.versions.kyuubi4paimon.get()
+val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
+
+dependencies {
+  implementation(project(":api")) {
+    exclude(group = "*")
+  }
+  implementation(project(":core")) {
+    exclude(group = "*")
+  }
+  implementation(project(":common")) {
+    exclude(group = "*")
+  }
+  implementation(project(":authorizations:authorization-common")) {
+    exclude(group = "*")
+  }
+  implementation(libs.bundles.log4j)
+  implementation(libs.commons.lang3)
+  implementation(libs.guava)
+  implementation(libs.javax.jaxb.api) {
+    exclude("*")
+  }
+  implementation(libs.javax.ws.rs.api)
+  implementation(libs.jettison)
+  implementation(libs.rome)
+  compileOnly(libs.lombok)
+
+  testImplementation(project(":core"))
+  testImplementation(project(":clients:client-java"))
+  testImplementation(project(":server"))
+  testImplementation(project(":catalogs:catalog-common"))
+  testImplementation(project(":integration-test-common", "testArtifacts"))
+  testImplementation(project(":authorizations:authorization-ranger"))
+  testImplementation(project(":authorizations:authorization-ranger", "testArtifacts"))
+  testImplementation(libs.junit.jupiter.api)
+  testImplementation(libs.mockito.core)
+  testImplementation(libs.testcontainers)
+  testRuntimeOnly(libs.junit.jupiter.engine)
+  testImplementation(libs.mysql.driver)
+  testImplementation(libs.postgresql.driver)
+  testImplementation(libs.ranger.intg) {
+    exclude("org.apache.hadoop", "hadoop-common")
+    exclude("org.apache.hive", "hive-storage-api")
+    exclude("org.apache.lucene")
+    exclude("org.apache.solr")
+    exclude("org.apache.kafka")
+    exclude("org.elasticsearch")
+    exclude("org.elasticsearch.client")
+    exclude("org.elasticsearch.plugin")
+    exclude("org.apache.ranger", "ranger-plugins-audit")
+    exclude("org.apache.ranger", "ranger-plugins-cred")
+    exclude("org.apache.ranger", "ranger-plugin-classloader")
+    exclude("net.java.dev.jna")
+    exclude("javax.ws.rs")
+    exclude("org.eclipse.jetty")
+  }
+  testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
+    exclude("org.apache.avro")
+    exclude("org.apache.hadoop")
+    exclude("org.apache.zookeeper")
+    exclude("io.dropwizard.metrics")
+    exclude("org.rocksdb")
+  }
+  testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") {
+    exclude("com.sun.jersey")
+  }
+  testImplementation(libs.hadoop3.client)
+  testImplementation(libs.hadoop3.common) {
+    exclude("com.sun.jersey")
+    exclude("javax.servlet", "servlet-api")
+  }
+  testImplementation(libs.hadoop3.hdfs) {
+    exclude("com.sun.jersey")
+    exclude("javax.servlet", "servlet-api")
+    exclude("io.netty")
+  }
+}
+
+tasks {
+  val runtimeJars by registering(Copy::class) {
+    from(configurations.runtimeClasspath)
+    into("build/libs")
+  }
+
+  val copyAuthorizationLibs by registering(Copy::class) {
+    dependsOn("jar", runtimeJars)
+    from("build/libs") {
+      exclude("guava-*.jar")
+      exclude("log4j-*.jar")
+      exclude("slf4j-*.jar")
+    }
+    into("$rootDir/distribution/package/authorizations/chain/libs")
+  }
+
+  register("copyLibAndConfig", Copy::class) {
+    dependsOn(copyAuthorizationLibs)
+  }
+
+  jar {
+    dependsOn(runtimeJars)
+  }
+}
+
+tasks.test {
+  doFirst {
+    environment("HADOOP_USER_NAME", "gravitino")
+  }
+  dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars", ":authorizations:authorization-ranger:jar", ":authorizations:authorization-ranger:runtimeJars")
+
+  val skipITs = project.hasProperty("skipITs")
+  if (skipITs) {
+    // Exclude integration tests
+    exclude("**/integration/test/**")
+  } else {
+    dependsOn(tasks.jar)
+  }
+}
diff --git a/...ization/mysql/TestMySQLAuthorization.java → ...orization/chain/ChainedAuthorization.java b/...ization/mysql/TestMySQLAuthorization.java → ...orization/chain/ChainedAuthorization.java
@@ -16,24 +16,27 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.connector.authorization.mysql;
+package org.apache.gravitino.authorization.chain;
 
 import java.util.Map;
 import org.apache.gravitino.connector.authorization.AuthorizationPlugin;
 import org.apache.gravitino.connector.authorization.BaseAuthorization;
 
-public class TestMySQLAuthorization extends BaseAuthorization<TestMySQLAuthorization> {
-
-  public TestMySQLAuthorization() {}
-
+/** Implementation of a Chained authorization in Gravitino. */
+public class ChainedAuthorization extends BaseAuthorization<ChainedAuthorization> {
   @Override
   public String shortName() {
-    return "mysql";
+    return "chain";
   }
 
   @Override
   public AuthorizationPlugin newPlugin(
       String metalake, String catalogProvider, Map<String, String> config) {
-    return new TestMySQLAuthorizationPlugin();
+    switch (catalogProvider) {
+      case "hive":
+        return new ChainedAuthorizationPlugin(metalake, catalogProvider, config);
+      default:
+        throw new IllegalArgumentException("Unknown catalog provider: " + catalogProvider);
+    }
   }
 }
diff --git a/...in/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java b/...in/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.authorization.chain;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.gravitino.Catalog;
+import org.apache.gravitino.MetadataObject;
+import org.apache.gravitino.authorization.Group;
+import org.apache.gravitino.authorization.MetadataObjectChange;
+import org.apache.gravitino.authorization.Owner;
+import org.apache.gravitino.authorization.Role;
+import org.apache.gravitino.authorization.RoleChange;
+import org.apache.gravitino.authorization.User;
+import org.apache.gravitino.authorization.common.AuthorizationProperties;
+import org.apache.gravitino.authorization.common.ChainedAuthorizationProperties;
+import org.apache.gravitino.connector.authorization.AuthorizationPlugin;
+import org.apache.gravitino.connector.authorization.BaseAuthorization;
+import org.apache.gravitino.exceptions.AuthorizationPluginException;
+import org.apache.gravitino.utils.IsolatedClassLoader;
+
+/** Chained authorization operations plugin class. <br> */
+public class ChainedAuthorizationPlugin implements AuthorizationPlugin {
+  private List<AuthorizationPlugin> plugins = Lists.newArrayList();
+  private final String metalake;
+
+  public ChainedAuthorizationPlugin(
+      String metalake, String catalogProvider, Map<String, String> config) {
+    this.metalake = metalake;
+    initPlugins(catalogProvider, config);
+  }
+
+  private void initPlugins(String catalogProvider, Map<String, String> properties) {
+    ChainedAuthorizationProperties chainedAuthzProperties =
+        new ChainedAuthorizationProperties(properties);
+    chainedAuthzProperties.validate();
+    // Validate the properties for each plugin
+    chainedAuthzProperties
+        .plugins()
+        .forEach(
+            pluginName -> {
+              Map<String, String> pluginProperties =
+                  chainedAuthzProperties.fetchAuthPluginProperties(pluginName);
+              String authzProvider = chainedAuthzProperties.getPluginProvider(pluginName);
+              AuthorizationProperties.validate(authzProvider, pluginProperties);
+            });
+    // Create the plugins
+    chainedAuthzProperties
+        .plugins()
+        .forEach(
+            pluginName -> {
+              String authzProvider = chainedAuthzProperties.getPluginProvider(pluginName);
+              Map<String, String> pluginConfig =
+                  chainedAuthzProperties.fetchAuthPluginProperties(pluginName);
+
+              ArrayList<String> libAndResourcesPaths = Lists.newArrayList();
+              BaseAuthorization.buildAuthorizationPkgPath(
+                      ImmutableMap.of(Catalog.AUTHORIZATION_PROVIDER, authzProvider))
+                  .ifPresent(libAndResourcesPaths::add);
+              IsolatedClassLoader classLoader =
+                  IsolatedClassLoader.buildClassLoader(libAndResourcesPaths);
+              try {
+                BaseAuthorization<?> authorization =
+                    BaseAuthorization.createAuthorization(classLoader, authzProvider);
+                AuthorizationPlugin authorizationPlugin =
+                    authorization.newPlugin(metalake, catalogProvider, pluginConfig);
+                plugins.add(authorizationPlugin);
+              } catch (Exception e) {
+                throw new RuntimeException(e);
+              }
+            });
+  }
+
+  @Override
+  public void close() throws IOException {
+    for (AuthorizationPlugin plugin : plugins) {
+      plugin.close();
+    }
+  }
+
+  @Override
+  public Boolean onMetadataUpdated(MetadataObjectChange... changes)
+      throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onMetadataUpdated(changes));
+  }
+
+  @Override
+  public Boolean onRoleCreated(Role role) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onRoleCreated(role));
+  }
+
+  @Override
+  public Boolean onRoleAcquired(Role role) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onRoleAcquired(role));
+  }
+
+  @Override
+  public Boolean onRoleDeleted(Role role) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onRoleDeleted(role));
+  }
+
+  @Override
+  public Boolean onRoleUpdated(Role role, RoleChange... changes)
+      throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onRoleUpdated(role, changes));
+  }
+
+  @Override
+  public Boolean onGrantedRolesToUser(List<Role> roles, User user)
+      throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onGrantedRolesToUser(roles, user));
+  }
+
+  @Override
+  public Boolean onRevokedRolesFromUser(List<Role> roles, User user)
+      throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onRevokedRolesFromUser(roles, user));
+  }
+
+  @Override
+  public Boolean onGrantedRolesToGroup(List<Role> roles, Group group)
+      throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onGrantedRolesToGroup(roles, group));
+  }
+
+  @Override
+  public Boolean onRevokedRolesFromGroup(List<Role> roles, Group group)
+      throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onRevokedRolesFromGroup(roles, group));
+  }
+
+  @Override
+  public Boolean onUserAdded(User user) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onUserAdded(user));
+  }
+
+  @Override
+  public Boolean onUserRemoved(User user) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onUserRemoved(user));
+  }
+
+  @Override
+  public Boolean onUserAcquired(User user) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onUserAcquired(user));
+  }
+
+  @Override
+  public Boolean onGroupAdded(Group group) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onGroupAdded(group));
+  }
+
+  @Override
+  public Boolean onGroupRemoved(Group group) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onGroupRemoved(group));
+  }
+
+  @Override
+  public Boolean onGroupAcquired(Group group) throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onGroupAcquired(group));
+  }
+
+  @Override
+  public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner)
+      throws AuthorizationPluginException {
+    return chainedAction(plugin -> plugin.onOwnerSet(metadataObject, preOwner, newOwner));
+  }
+
+  private Boolean chainedAction(Function<AuthorizationPlugin, Boolean> action) {
+    for (AuthorizationPlugin plugin : plugins) {
+      if (!action.apply(plugin)) {
+        return false;
+      }
+    }
+    return true;
+  }
+}