From 3e7e55010a6c4d61ab3233542b9869226c0272e3 Mon Sep 17 00:00:00 2001 From: Xun Date: Wed, 25 Dec 2024 15:38:24 +0800 Subject: [PATCH 1/7] [#5775] feat(auth): Chain authorization plugin framework (#5786) ### What changes were proposed in this pull request? 1. Add Chain auth plugin module 1. Add auth common module 3. Add Chain authorization Ranger Hive and Ranger HDFS ITs ### Why are the changes needed? Fix: #5775 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Add ITs --- .../access-control-integration-test.yml | 9 +- .../authorization-chain/build.gradle.kts | 146 +++++++ .../chain/ChainedAuthorization.java | 17 +- .../chain/ChainedAuthorizationPlugin.java | 197 +++++++++ ...nector.authorization.AuthorizationProvider | 19 + .../test/TestChainedAuthorizationIT.java | 374 ++++++++++++++++++ .../src/test/resources/log4j2.properties | 73 ++++ .../ranger-spark-security.xml.template | 45 +++ .../authorization-common/build.gradle.kts | 62 +++ .../common/AuthorizationProperties.java | 54 +++ .../ChainedAuthorizationProperties.java} | 36 +- .../common}/JdbcAuthorizationProperties.java | 28 +- .../common/PathBasedMetadataObject.java} | 33 +- .../common/PathBasedSecurableObject.java} | 6 +- .../RangerAuthorizationProperties.java | 16 +- .../TestChainedAuthorizationProperties.java} | 84 +++- .../TestRangerAuthorizationProperties.java | 51 ++- .../authorization-jdbc/build.gradle.kts | 4 +- .../jdbc/JdbcAuthorizationPlugin.java | 4 +- .../jdbc/JdbcAuthorizationPluginTest.java | 1 + .../authorization-ranger/build.gradle.kts | 27 +- .../ranger/RangerAuthorization.java | 10 +- .../ranger/RangerAuthorizationHDFSPlugin.java | 195 +++++++-- .../RangerAuthorizationHadoopSQLPlugin.java | 44 +-- .../ranger/RangerAuthorizationPlugin.java | 5 +- .../ranger/RangerHadoopSQLMetadataObject.java | 2 +- .../authorization/ranger/RangerHelper.java | 2 +- .../ranger/RangerPrivileges.java | 26 ++ .../test/RangerAuthorizationHDFSPluginIT.java | 15 +- .../integration/test/RangerBaseE2EIT.java | 11 +- .../integration/test/RangerFilesetIT.java | 2 +- .../integration/test/RangerHiveE2EIT.java | 12 +- .../ranger/integration/test/RangerITEnv.java | 11 +- .../integration/test/RangerIcebergE2EIT.java | 12 +- .../integration/test/RangerPaimonE2EIT.java | 12 +- authorizations/build.gradle.kts | 18 +- build.gradle.kts | 8 +- .../integration/test/ProxyCatalogHiveIT.java | 18 - .../gravitino/catalog/CatalogManager.java | 35 +- .../gravitino/connector/BaseCatalog.java | 35 +- .../authorization/BaseAuthorization.java | 72 ++++ .../authorization/TestAuthorization.java | 62 +-- .../ranger/TestRangerAuthorization.java | 18 +- .../TestRangerAuthorizationHDFSPlugin.java} | 4 +- ...stRangerAuthorizationHadoopSQLPlugin.java} | 2 +- ...nector.authorization.AuthorizationProvider | 3 +- integration-test-common/build.gradle.kts | 5 +- .../integration/test/util/BaseIT.java | 38 ++ settings.gradle.kts | 2 +- 49 files changed, 1630 insertions(+), 335 deletions(-) create mode 100644 authorizations/authorization-chain/build.gradle.kts rename core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java => authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java (71%) create mode 100644 authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java create mode 100644 authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider create mode 100644 authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java create mode 100644 authorizations/authorization-chain/src/test/resources/log4j2.properties create mode 100644 authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template create mode 100644 authorizations/authorization-common/build.gradle.kts create mode 100644 authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java => authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java} (87%) rename authorizations/{authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc => authorization-common/src/main/java/org/apache/gravitino/authorization/common}/JdbcAuthorizationProperties.java (73%) rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java => authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java} (64%) rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java => authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java} (89%) rename authorizations/{authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger => authorization-common/src/main/java/org/apache/gravitino/authorization/common}/RangerAuthorizationProperties.java (90%) rename authorizations/{authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java => authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java} (75%) rename authorizations/{authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger => authorization-common/src/test/java/org/apache/gravitino/authorization/common}/TestRangerAuthorizationProperties.java (72%) rename core/src/test/java/org/apache/gravitino/connector/authorization/{mysql/TestMySQLAuthorizationPlugin.java => ranger/TestRangerAuthorizationHDFSPlugin.java} (95%) rename core/src/test/java/org/apache/gravitino/connector/authorization/ranger/{TestRangerAuthorizationPlugin.java => TestRangerAuthorizationHadoopSQLPlugin.java} (97%) diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index 6997eaf9a4c..dc8acd60678 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -87,12 +87,9 @@ jobs: - name: Authorization Integration Test (JDK${{ matrix.java-version }}) id: integrationTest run: | - ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test - ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test - ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test - ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test - ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test - ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-jdbc:test + ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test + ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test + ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/authorizations/authorization-chain/build.gradle.kts b/authorizations/authorization-chain/build.gradle.kts new file mode 100644 index 00000000000..d5cd160742c --- /dev/null +++ b/authorizations/authorization-chain/build.gradle.kts @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "authorization-chain" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() +val sparkVersion: String = libs.versions.spark35.get() +val kyuubiVersion: String = libs.versions.kyuubi4paimon.get() +val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".") + +dependencies { + implementation(project(":api")) { + exclude(group = "*") + } + implementation(project(":core")) { + exclude(group = "*") + } + implementation(project(":common")) { + exclude(group = "*") + } + implementation(project(":authorizations:authorization-common")) { + exclude(group = "*") + } + implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.javax.jaxb.api) { + exclude("*") + } + implementation(libs.javax.ws.rs.api) + implementation(libs.jettison) + implementation(libs.rome) + compileOnly(libs.lombok) + + testImplementation(project(":core")) + testImplementation(project(":clients:client-java")) + testImplementation(project(":server")) + testImplementation(project(":catalogs:catalog-common")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":authorizations:authorization-ranger")) + testImplementation(project(":authorizations:authorization-ranger", "testArtifacts")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.mockito.core) + testImplementation(libs.testcontainers) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) + testImplementation(libs.ranger.intg) { + exclude("org.apache.hadoop", "hadoop-common") + exclude("org.apache.hive", "hive-storage-api") + exclude("org.apache.lucene") + exclude("org.apache.solr") + exclude("org.apache.kafka") + exclude("org.elasticsearch") + exclude("org.elasticsearch.client") + exclude("org.elasticsearch.plugin") + exclude("org.apache.ranger", "ranger-plugins-audit") + exclude("org.apache.ranger", "ranger-plugins-cred") + exclude("org.apache.ranger", "ranger-plugin-classloader") + exclude("net.java.dev.jna") + exclude("javax.ws.rs") + exclude("org.eclipse.jetty") + } + testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") + testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { + exclude("org.apache.avro") + exclude("org.apache.hadoop") + exclude("org.apache.zookeeper") + exclude("io.dropwizard.metrics") + exclude("org.rocksdb") + } + testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") { + exclude("com.sun.jersey") + } + testImplementation(libs.hadoop3.client) + testImplementation(libs.hadoop3.common) { + exclude("com.sun.jersey") + exclude("javax.servlet", "servlet-api") + } + testImplementation(libs.hadoop3.hdfs) { + exclude("com.sun.jersey") + exclude("javax.servlet", "servlet-api") + exclude("io.netty") + } +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } + + val copyAuthorizationLibs by registering(Copy::class) { + dependsOn("jar", runtimeJars) + from("build/libs") { + exclude("guava-*.jar") + exclude("log4j-*.jar") + exclude("slf4j-*.jar") + } + into("$rootDir/distribution/package/authorizations/chain/libs") + } + + register("copyLibAndConfig", Copy::class) { + dependsOn(copyAuthorizationLibs) + } + + jar { + dependsOn(runtimeJars) + } +} + +tasks.test { + doFirst { + environment("HADOOP_USER_NAME", "gravitino") + } + dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars", ":authorizations:authorization-ranger:jar", ":authorizations:authorization-ranger:runtimeJars") + + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/test/**") + } else { + dependsOn(tasks.jar) + } +} diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java similarity index 71% rename from core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java rename to authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java index e8d747da11f..5f8c9834750 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorization.java +++ b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorization.java @@ -16,24 +16,27 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.connector.authorization.mysql; +package org.apache.gravitino.authorization.chain; import java.util.Map; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; -public class TestMySQLAuthorization extends BaseAuthorization { - - public TestMySQLAuthorization() {} - +/** Implementation of a Chained authorization in Gravitino. */ +public class ChainedAuthorization extends BaseAuthorization { @Override public String shortName() { - return "mysql"; + return "chain"; } @Override public AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map config) { - return new TestMySQLAuthorizationPlugin(); + switch (catalogProvider) { + case "hive": + return new ChainedAuthorizationPlugin(metalake, catalogProvider, config); + default: + throw new IllegalArgumentException("Unknown catalog provider: " + catalogProvider); + } } } diff --git a/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java new file mode 100644 index 00000000000..120c355db06 --- /dev/null +++ b/authorizations/authorization-chain/src/main/java/org/apache/gravitino/authorization/chain/ChainedAuthorizationPlugin.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.chain; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.MetadataObjectChange; +import org.apache.gravitino.authorization.Owner; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.AuthorizationProperties; +import org.apache.gravitino.authorization.common.ChainedAuthorizationProperties; +import org.apache.gravitino.connector.authorization.AuthorizationPlugin; +import org.apache.gravitino.connector.authorization.BaseAuthorization; +import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.apache.gravitino.utils.IsolatedClassLoader; + +/** Chained authorization operations plugin class.
*/ +public class ChainedAuthorizationPlugin implements AuthorizationPlugin { + private List plugins = Lists.newArrayList(); + private final String metalake; + + public ChainedAuthorizationPlugin( + String metalake, String catalogProvider, Map config) { + this.metalake = metalake; + initPlugins(catalogProvider, config); + } + + private void initPlugins(String catalogProvider, Map properties) { + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + chainedAuthzProperties.validate(); + // Validate the properties for each plugin + chainedAuthzProperties + .plugins() + .forEach( + pluginName -> { + Map pluginProperties = + chainedAuthzProperties.fetchAuthPluginProperties(pluginName); + String authzProvider = chainedAuthzProperties.getPluginProvider(pluginName); + AuthorizationProperties.validate(authzProvider, pluginProperties); + }); + // Create the plugins + chainedAuthzProperties + .plugins() + .forEach( + pluginName -> { + String authzProvider = chainedAuthzProperties.getPluginProvider(pluginName); + Map pluginConfig = + chainedAuthzProperties.fetchAuthPluginProperties(pluginName); + + ArrayList libAndResourcesPaths = Lists.newArrayList(); + BaseAuthorization.buildAuthorizationPkgPath( + ImmutableMap.of(Catalog.AUTHORIZATION_PROVIDER, authzProvider)) + .ifPresent(libAndResourcesPaths::add); + IsolatedClassLoader classLoader = + IsolatedClassLoader.buildClassLoader(libAndResourcesPaths); + try { + BaseAuthorization authorization = + BaseAuthorization.createAuthorization(classLoader, authzProvider); + AuthorizationPlugin authorizationPlugin = + authorization.newPlugin(metalake, catalogProvider, pluginConfig); + plugins.add(authorizationPlugin); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public void close() throws IOException { + for (AuthorizationPlugin plugin : plugins) { + plugin.close(); + } + } + + @Override + public Boolean onMetadataUpdated(MetadataObjectChange... changes) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onMetadataUpdated(changes)); + } + + @Override + public Boolean onRoleCreated(Role role) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleCreated(role)); + } + + @Override + public Boolean onRoleAcquired(Role role) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleAcquired(role)); + } + + @Override + public Boolean onRoleDeleted(Role role) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleDeleted(role)); + } + + @Override + public Boolean onRoleUpdated(Role role, RoleChange... changes) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRoleUpdated(role, changes)); + } + + @Override + public Boolean onGrantedRolesToUser(List roles, User user) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGrantedRolesToUser(roles, user)); + } + + @Override + public Boolean onRevokedRolesFromUser(List roles, User user) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRevokedRolesFromUser(roles, user)); + } + + @Override + public Boolean onGrantedRolesToGroup(List roles, Group group) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGrantedRolesToGroup(roles, group)); + } + + @Override + public Boolean onRevokedRolesFromGroup(List roles, Group group) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onRevokedRolesFromGroup(roles, group)); + } + + @Override + public Boolean onUserAdded(User user) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onUserAdded(user)); + } + + @Override + public Boolean onUserRemoved(User user) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onUserRemoved(user)); + } + + @Override + public Boolean onUserAcquired(User user) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onUserAcquired(user)); + } + + @Override + public Boolean onGroupAdded(Group group) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGroupAdded(group)); + } + + @Override + public Boolean onGroupRemoved(Group group) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGroupRemoved(group)); + } + + @Override + public Boolean onGroupAcquired(Group group) throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onGroupAcquired(group)); + } + + @Override + public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) + throws AuthorizationPluginException { + return chainedAction(plugin -> plugin.onOwnerSet(metadataObject, preOwner, newOwner)); + } + + private Boolean chainedAction(Function action) { + for (AuthorizationPlugin plugin : plugins) { + if (!action.apply(plugin)) { + return false; + } + } + return true; + } +} diff --git a/authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider b/authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider new file mode 100644 index 00000000000..f4bea1086db --- /dev/null +++ b/authorizations/authorization-chain/src/main/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.authorization.chain.ChainedAuthorization \ No newline at end of file diff --git a/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java new file mode 100644 index 00000000000..74ad99aa7f9 --- /dev/null +++ b/authorizations/authorization-chain/src/test/java/org/apache/gravitino/authorization/chain/integration/test/TestChainedAuthorizationIT.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.chain.integration.test; + +import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; +import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.Configs; +import org.apache.gravitino.auth.AuthConstants; +import org.apache.gravitino.auth.AuthenticatorType; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.common.ChainedAuthorizationProperties; +import org.apache.gravitino.authorization.ranger.integration.test.RangerBaseE2EIT; +import org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv; +import org.apache.gravitino.catalog.hive.HiveConstants; +import org.apache.gravitino.exceptions.UserAlreadyExistsException; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.container.RangerContainer; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.kyuubi.plugin.spark.authz.AccessControlException; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestChainedAuthorizationIT extends RangerBaseE2EIT { + private static final Logger LOG = LoggerFactory.getLogger(TestChainedAuthorizationIT.class); + private static String DEFAULT_FS; + private FileSystem fileSystem; + + @BeforeAll + public void startIntegrationTest() throws Exception { + metalakeName = GravitinoITUtils.genRandomName("metalake").toLowerCase(); + // Enable Gravitino Authorization mode + Map configs = Maps.newHashMap(); + configs.put(Configs.ENABLE_AUTHORIZATION.getKey(), String.valueOf(true)); + configs.put(Configs.SERVICE_ADMINS.getKey(), RangerITEnv.HADOOP_USER_NAME); + configs.put(Configs.AUTHENTICATORS.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase()); + configs.put("SimpleAuthUserName", AuthConstants.ANONYMOUS_USER); + registerCustomConfigs(configs); + + super.startIntegrationTest(); + RangerITEnv.init(RangerBaseE2EIT.metalakeName, false); + RangerITEnv.startHiveRangerContainer(); + + HIVE_METASTORE_URIS = + String.format( + "thrift://%s:%d", + containerSuite.getHiveRangerContainer().getContainerIpAddress(), + HiveContainer.HIVE_METASTORE_PORT); + + generateRangerSparkSecurityXML("authorization-chain"); + + DEFAULT_FS = + String.format( + "hdfs://%s:%d/user/hive/warehouse", + containerSuite.getHiveRangerContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT); + BaseIT.runInEnv( + "HADOOP_USER_NAME", + AuthConstants.ANONYMOUS_USER, + () -> { + sparkSession = + SparkSession.builder() + .master("local[1]") + .appName("Ranger Hive E2E integration test") + .config("hive.metastore.uris", HIVE_METASTORE_URIS) + .config("spark.sql.warehouse.dir", DEFAULT_FS) + .config("spark.sql.storeAssignmentPolicy", "LEGACY") + .config("mapreduce.input.fileinputformat.input.dir.recursive", "true") + .config( + "spark.sql.extensions", + "org.apache.kyuubi.plugin.spark.authz.ranger.RangerSparkExtension") + .enableHiveSupport() + .getOrCreate(); + sparkSession.sql(SQL_SHOW_DATABASES); // must be called to activate the Spark session + }); + createMetalake(); + createCatalog(); + + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", DEFAULT_FS); + fileSystem = FileSystem.get(conf); + + RangerITEnv.cleanup(); + try { + metalake.addUser(System.getenv(HADOOP_USER_NAME)); + } catch (UserAlreadyExistsException e) { + LOG.error("Failed to add user: {}", System.getenv(HADOOP_USER_NAME), e); + } + } + + @AfterAll + public void stop() throws IOException { + if (client != null) { + Arrays.stream(catalog.asSchemas().listSchemas()) + .filter(schema -> !schema.equals("default")) + .forEach( + (schema -> { + catalog.asSchemas().dropSchema(schema, false); + })); + Arrays.stream(metalake.listCatalogs()) + .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); + client.disableMetalake(metalakeName); + client.dropMetalake(metalakeName); + } + if (fileSystem != null) { + fileSystem.close(); + } + try { + closer.close(); + } catch (Exception e) { + LOG.error("Failed to close CloseableGroup", e); + } + client = null; + RangerITEnv.cleanup(); + } + + private String storageLocation(String dirName) { + return DEFAULT_FS + "/" + dirName; + } + + @Test + public void testCreateSchemaInCatalog() throws IOException { + // Choose a catalog + useCatalog(); + + // First, fail to create the schema + Exception accessControlException = + Assertions.assertThrows(Exception.class, () -> sparkSession.sql(SQL_CREATE_SCHEMA)); + Assertions.assertTrue( + accessControlException + .getMessage() + .contains( + String.format( + "Permission denied: user [%s] does not have [create] privilege", + AuthConstants.ANONYMOUS_USER)) + || accessControlException + .getMessage() + .contains( + String.format( + "Permission denied: user=%s, access=WRITE", AuthConstants.ANONYMOUS_USER))); + Path schemaPath = new Path(storageLocation(schemaName + ".db")); + Assertions.assertFalse(fileSystem.exists(schemaPath)); + FileStatus fileStatus = fileSystem.getFileStatus(new Path(DEFAULT_FS)); + Assertions.assertEquals(System.getenv(HADOOP_USER_NAME), fileStatus.getOwner()); + + // Second, grant the `CREATE_SCHEMA` role + String roleName = currentFunName(); + SecurableObject securableObject = + SecurableObjects.ofCatalog( + catalogName, Lists.newArrayList(Privileges.CreateSchema.allow())); + metalake.createRole(roleName, Collections.emptyMap(), Lists.newArrayList(securableObject)); + metalake.grantRolesToUser(Lists.newArrayList(roleName), AuthConstants.ANONYMOUS_USER); + waitForUpdatingPolicies(); + + // Third, succeed to create the schema + sparkSession.sql(SQL_CREATE_SCHEMA); + Assertions.assertTrue(fileSystem.exists(schemaPath)); + FileStatus fsSchema = fileSystem.getFileStatus(schemaPath); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, fsSchema.getOwner()); + + // Fourth, fail to create the table + Assertions.assertThrows(AccessControlException.class, () -> sparkSession.sql(SQL_CREATE_TABLE)); + + // Clean up + catalog.asSchemas().dropSchema(schemaName, false); + metalake.deleteRole(roleName); + waitForUpdatingPolicies(); + + Exception accessControlException2 = + Assertions.assertThrows(Exception.class, () -> sparkSession.sql(SQL_CREATE_SCHEMA)); + Assertions.assertTrue( + accessControlException2 + .getMessage() + .contains( + String.format( + "Permission denied: user [%s] does not have [create] privilege", + AuthConstants.ANONYMOUS_USER)) + || accessControlException2 + .getMessage() + .contains( + String.format( + "Permission denied: user=%s, access=WRITE", AuthConstants.ANONYMOUS_USER))); + } + + @Override + public void createCatalog() { + Map catalogConf = new HashMap<>(); + catalogConf.put(HiveConstants.METASTORE_URIS, HIVE_METASTORE_URIS); + catalogConf.put(IMPERSONATION_ENABLE, "true"); + catalogConf.put(Catalog.AUTHORIZATION_PROVIDER, "chain"); + catalogConf.put(ChainedAuthorizationProperties.CHAIN_PLUGINS_PROPERTIES_KEY, "hive1,hdfs1"); + catalogConf.put("authorization.chain.hive1.provider", "ranger"); + catalogConf.put("authorization.chain.hive1.ranger.auth.type", RangerContainer.authType); + catalogConf.put("authorization.chain.hive1.ranger.admin.url", RangerITEnv.RANGER_ADMIN_URL); + catalogConf.put("authorization.chain.hive1.ranger.username", RangerContainer.rangerUserName); + catalogConf.put("authorization.chain.hive1.ranger.password", RangerContainer.rangerPassword); + catalogConf.put("authorization.chain.hive1.ranger.service.type", "HadoopSQL"); + catalogConf.put( + "authorization.chain.hive1.ranger.service.name", RangerITEnv.RANGER_HIVE_REPO_NAME); + catalogConf.put("authorization.chain.hdfs1.provider", "ranger"); + catalogConf.put("authorization.chain.hdfs1.ranger.auth.type", RangerContainer.authType); + catalogConf.put("authorization.chain.hdfs1.ranger.admin.url", RangerITEnv.RANGER_ADMIN_URL); + catalogConf.put("authorization.chain.hdfs1.ranger.username", RangerContainer.rangerUserName); + catalogConf.put("authorization.chain.hdfs1.ranger.password", RangerContainer.rangerPassword); + catalogConf.put("authorization.chain.hdfs1.ranger.service.type", "HDFS"); + catalogConf.put( + "authorization.chain.hdfs1.ranger.service.name", RangerITEnv.RANGER_HDFS_REPO_NAME); + + metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "hive", "comment", catalogConf); + catalog = metalake.loadCatalog(catalogName); + LOG.info("Catalog created: {}", catalog); + } + + @Test + public void testCreateSchema() throws InterruptedException { + // TODO + } + + @Test + void testCreateTable() throws InterruptedException { + // TODO + } + + @Test + void testReadWriteTableWithMetalakeLevelRole() throws InterruptedException { + // TODO + } + + @Test + void testReadWriteTableWithTableLevelRole() throws InterruptedException { + // TODO + } + + @Test + void testReadOnlyTable() throws InterruptedException { + // TODO + } + + @Test + void testWriteOnlyTable() throws InterruptedException { + // TODO + } + + @Test + void testCreateAllPrivilegesRole() throws InterruptedException { + // TODO + } + + @Test + void testDeleteAndRecreateRole() throws InterruptedException { + // TODO + } + + @Test + void testDeleteAndRecreateMetadataObject() throws InterruptedException { + // TODO + } + + @Test + void testRenameMetadataObject() throws InterruptedException { + // TODO + } + + @Test + void testRenameMetadataObjectPrivilege() throws InterruptedException { + // TODO + } + + @Test + void testChangeOwner() throws InterruptedException { + // TODO + } + + @Test + void testAllowUseSchemaPrivilege() throws InterruptedException { + // TODO + } + + @Test + void testDenyPrivileges() throws InterruptedException { + // TODO + } + + @Test + void testGrantPrivilegesForMetalake() throws InterruptedException { + // TODO + } + + @Override + protected void checkTableAllPrivilegesExceptForCreating() { + // TODO + } + + @Override + protected void checkUpdateSQLWithReadWritePrivileges() { + // TODO + } + + @Override + protected void checkUpdateSQLWithReadPrivileges() { + // TODO + } + + @Override + protected void checkUpdateSQLWithWritePrivileges() { + // TODO + } + + @Override + protected void checkDeleteSQLWithReadWritePrivileges() { + // TODO + } + + @Override + protected void checkDeleteSQLWithReadPrivileges() { + // TODO + } + + @Override + protected void checkDeleteSQLWithWritePrivileges() { + // TODO + } + + @Override + protected void useCatalog() { + // TODO + } + + @Override + protected void checkWithoutPrivileges() { + // TODO + } + + @Override + protected void testAlterTable() { + // TODO + } +} diff --git a/authorizations/authorization-chain/src/test/resources/log4j2.properties b/authorizations/authorization-chain/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..2a46c57ec2f --- /dev/null +++ b/authorizations/authorization-chain/src/test/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/authorization-chain-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off diff --git a/authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template b/authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template new file mode 100644 index 00000000000..eb7f2b5e811 --- /dev/null +++ b/authorizations/authorization-chain/src/test/resources/ranger-spark-security.xml.template @@ -0,0 +1,45 @@ + + + + ranger.plugin.spark.policy.rest.url + __REPLACE__RANGER_ADMIN_URL + + + + ranger.plugin.spark.service.name + __REPLACE__RANGER_HIVE_REPO_NAME + + + + ranger.plugin.spark.policy.cache.dir + /tmp/policycache + + + + ranger.plugin.spark.policy.pollIntervalMs + 500 + + + + ranger.plugin.spark.policy.source.impl + org.apache.ranger.admin.client.RangerAdminRESTClient + + + \ No newline at end of file diff --git a/authorizations/authorization-common/build.gradle.kts b/authorizations/authorization-common/build.gradle.kts new file mode 100644 index 00000000000..ba64510f2ce --- /dev/null +++ b/authorizations/authorization-common/build.gradle.kts @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "authorization-chain" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + implementation(project(":api")) { + exclude(group = "*") + } + implementation(project(":core")) { + exclude(group = "*") + } + implementation(project(":common")) { + exclude(group = "*") + } + implementation(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.javax.jaxb.api) { + exclude("*") + } + implementation(libs.javax.ws.rs.api) + implementation(libs.jettison) + implementation(libs.rome) + compileOnly(libs.lombok) + + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.mockito.core) + testImplementation(libs.testcontainers) + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks.test { + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/test/**") + } else { + dependsOn(tasks.jar) + } +} diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java new file mode 100644 index 00000000000..3005cc5f3e9 --- /dev/null +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.common; + +import java.util.Map; +import java.util.stream.Collectors; + +public abstract class AuthorizationProperties { + protected Map properties; + + public AuthorizationProperties(Map properties) { + this.properties = + properties.entrySet().stream() + .filter(entry -> entry.getKey().startsWith(getPropertiesPrefix())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + abstract String getPropertiesPrefix(); + + abstract void validate(); + + public static void validate(String type, Map properties) { + switch (type) { + case "ranger": + RangerAuthorizationProperties rangerAuthorizationProperties = + new RangerAuthorizationProperties(properties); + rangerAuthorizationProperties.validate(); + break; + case "chain": + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + chainedAuthzProperties.validate(); + break; + default: + throw new IllegalArgumentException("Unsupported authorization properties type: " + type); + } + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java similarity index 87% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java index edaa375747a..7e5aea0ca29 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/ChainAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/ChainedAuthorizationProperties.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -29,7 +29,7 @@ import java.util.stream.Collectors; /** - * The properties for Chain authorization plugin.
+ * The properties for Chained authorization plugin.
*
* Configuration Example:
* "authorization.chain.plugins" = "hive1,hdfs1"
@@ -48,16 +48,32 @@ * "authorization.chain.hdfs1.ranger.username" = "admin";
* "authorization.chain.hdfs1.ranger.password" = "admin";
*/ -public class ChainAuthorizationProperties { - public static final String PLUGINS_SPLITTER = ","; - /** Chain authorization plugin names */ +public class ChainedAuthorizationProperties extends AuthorizationProperties { + private static final String PLUGINS_SPLITTER = ","; + /** Chained authorization plugin names */ public static final String CHAIN_PLUGINS_PROPERTIES_KEY = "authorization.chain.plugins"; - /** Chain authorization plugin provider */ + /** Chained authorization plugin provider */ public static final String CHAIN_PROVIDER = "authorization.chain.*.provider"; - static Map fetchAuthPluginProperties( - String pluginName, Map properties) { + public ChainedAuthorizationProperties(Map properties) { + super(properties); + } + + @Override + public String getPropertiesPrefix() { + return "authorization.chain"; + } + + public String getPluginProvider(String pluginName) { + return properties.get(getPropertiesPrefix() + "." + pluginName + ".provider"); + } + + public List plugins() { + return Arrays.asList(properties.get(CHAIN_PLUGINS_PROPERTIES_KEY).split(PLUGINS_SPLITTER)); + } + + public Map fetchAuthPluginProperties(String pluginName) { Preconditions.checkArgument( properties.containsKey(CHAIN_PLUGINS_PROPERTIES_KEY) && properties.get(CHAIN_PLUGINS_PROPERTIES_KEY) != null, @@ -93,13 +109,15 @@ static Map fetchAuthPluginProperties( return resultProperties; } - public static void validate(Map properties) { + @Override + public void validate() { Preconditions.checkArgument( properties.containsKey(CHAIN_PLUGINS_PROPERTIES_KEY), String.format("%s is required", CHAIN_PLUGINS_PROPERTIES_KEY)); List pluginNames = Arrays.stream(properties.get(CHAIN_PLUGINS_PROPERTIES_KEY).split(PLUGINS_SPLITTER)) .map(String::trim) + .filter(v -> !v.isEmpty()) .collect(Collectors.toList()); Preconditions.checkArgument( !pluginNames.isEmpty(), diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java similarity index 73% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java index b13504fd2fd..9a5e7c6cc97 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java @@ -16,29 +16,39 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.jdbc; +package org.apache.gravitino.authorization.common; import java.util.Map; /** The properties for JDBC authorization plugin. */ -public class JdbcAuthorizationProperties { +public class JdbcAuthorizationProperties extends AuthorizationProperties { private static final String CONFIG_PREFIX = "authorization.jdbc."; public static final String JDBC_PASSWORD = CONFIG_PREFIX + "password"; public static final String JDBC_USERNAME = CONFIG_PREFIX + "username"; public static final String JDBC_URL = CONFIG_PREFIX + "url"; public static final String JDBC_DRIVER = CONFIG_PREFIX + "driver"; - public static void validate(Map properties) { - String errorMsg = "%s is required"; - check(properties, JDBC_URL, errorMsg); - check(properties, JDBC_USERNAME, errorMsg); - check(properties, JDBC_PASSWORD, errorMsg); - check(properties, JDBC_DRIVER, errorMsg); + public JdbcAuthorizationProperties(Map properties) { + super(properties); } - private static void check(Map properties, String key, String errorMsg) { + private void check(String key, String errorMsg) { if (!properties.containsKey(key) && properties.get(key) != null) { throw new IllegalArgumentException(String.format(errorMsg, key)); } } + + @Override + String getPropertiesPrefix() { + return CONFIG_PREFIX; + } + + @Override + public void validate() { + String errorMsg = "%s is required"; + check(JDBC_URL, errorMsg); + check(JDBC_USERNAME, errorMsg); + check(JDBC_PASSWORD, errorMsg); + check(JDBC_DRIVER, errorMsg); + } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java similarity index 64% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java index 77523464162..ed67b1cc0fc 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseMetadataObject.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedMetadataObject.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -25,10 +25,10 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.AuthorizationMetadataObject; -public class RangerPathBaseMetadataObject implements AuthorizationMetadataObject { +public class PathBasedMetadataObject implements AuthorizationMetadataObject { /** - * The type of object in the Ranger system. Every type will map one kind of the entity of the - * Gravitino type system. + * The type of metadata object in the underlying system. Every type will map one kind of the + * entity of the Gravitino type system. */ public enum Type implements AuthorizationMetadataObject.Type { /** A path is mapped the path of storages like HDFS, S3 etc. */ @@ -42,24 +42,13 @@ public enum Type implements AuthorizationMetadataObject.Type { public MetadataObject.Type metadataObjectType() { return metadataType; } - - public static RangerHadoopSQLMetadataObject.Type fromMetadataType( - MetadataObject.Type metadataType) { - for (RangerHadoopSQLMetadataObject.Type type : RangerHadoopSQLMetadataObject.Type.values()) { - if (type.metadataObjectType() == metadataType) { - return type; - } - } - throw new IllegalArgumentException( - "No matching RangerMetadataObject.Type for " + metadataType); - } } private final String path; private final AuthorizationMetadataObject.Type type; - public RangerPathBaseMetadataObject(String path, AuthorizationMetadataObject.Type type) { + public PathBasedMetadataObject(String path, AuthorizationMetadataObject.Type type) { this.path = path; this.type = type; } @@ -89,18 +78,20 @@ public AuthorizationMetadataObject.Type type() { public void validateAuthorizationMetadataObject() throws IllegalArgumentException { List names = names(); Preconditions.checkArgument( - names != null && !names.isEmpty(), "Cannot create a Ranger metadata object with no names"); + names != null && !names.isEmpty(), + "Cannot create a path based metadata object with no names"); Preconditions.checkArgument( names.size() == 1, - "Cannot create a Ranger metadata object with the name length which is 1"); + "Cannot create a path based metadata object with the name length which is 1"); Preconditions.checkArgument( - type != null, "Cannot create a Ranger metadata object with no type"); + type != null, "Cannot create a path based metadata object with no type"); Preconditions.checkArgument( - type == RangerPathBaseMetadataObject.Type.PATH, "it must be the PATH type"); + type == PathBasedMetadataObject.Type.PATH, "it must be the PATH type"); for (String name : names) { - Preconditions.checkArgument(name != null, "Cannot create a metadata object with null name"); + Preconditions.checkArgument( + name != null, "Cannot create a path based metadata object with null name"); } } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java similarity index 89% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java index bd2c73fdaef..6712cdf0e3d 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPathBaseSecurableObject.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/PathBasedSecurableObject.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.collect.ImmutableList; import java.util.List; @@ -25,12 +25,12 @@ import org.apache.gravitino.authorization.AuthorizationPrivilege; import org.apache.gravitino.authorization.AuthorizationSecurableObject; -public class RangerPathBaseSecurableObject extends RangerPathBaseMetadataObject +public class PathBasedSecurableObject extends PathBasedMetadataObject implements AuthorizationSecurableObject { private final List privileges; - public RangerPathBaseSecurableObject( + public PathBasedSecurableObject( String path, AuthorizationMetadataObject.Type type, Set privileges) { super(path, type); this.privileges = ImmutableList.copyOf(privileges); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/RangerAuthorizationProperties.java similarity index 90% rename from authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/RangerAuthorizationProperties.java index e7fee3088f6..73af3bc377e 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/RangerAuthorizationProperties.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.base.Preconditions; import java.util.Map; /** The properties for Ranger authorization plugin. */ -public class RangerAuthorizationProperties { +public class RangerAuthorizationProperties extends AuthorizationProperties { /** Ranger admin web URIs */ public static final String RANGER_ADMIN_URL = "authorization.ranger.admin.url"; @@ -46,7 +46,17 @@ public class RangerAuthorizationProperties { */ public static final String RANGER_PASSWORD = "authorization.ranger.password"; - public static void validate(Map properties) { + public RangerAuthorizationProperties(Map properties) { + super(properties); + } + + @Override + public String getPropertiesPrefix() { + return "authorization.ranger"; + } + + @Override + public void validate() { Preconditions.checkArgument( properties.containsKey(RANGER_ADMIN_URL), String.format("%s is required", RANGER_ADMIN_URL)); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java similarity index 75% rename from authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java rename to authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java index 5d19f234093..7c0cccc738c 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestChainAuthorizationProperties.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestChainedAuthorizationProperties.java @@ -16,21 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; -import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; import com.google.common.collect.Maps; import java.util.HashMap; import java.util.Map; -import org.apache.gravitino.catalog.hive.HiveConstants; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class TestChainAuthorizationProperties { +public class TestChainedAuthorizationProperties { + static final String METASTORE_URIS = "metastore.uris"; + public static final String IMPERSONATION_ENABLE = "impersonation-enable"; + @Test - void testChainOnePlugin() { + void testChainedOnePlugin() { Map properties = Maps.newHashMap(); properties.put("authorization.chain.plugins", "hive1"); properties.put("authorization.chain.hive1.provider", "ranger"); @@ -40,13 +41,15 @@ void testChainOnePlugin() { properties.put("authorization.chain.hive1.ranger.password", "admin"); properties.put("authorization.chain.hive1.ranger.service.type", "hive"); properties.put("authorization.chain.hive1.ranger.service.name", "hiveDev"); - Assertions.assertDoesNotThrow(() -> ChainAuthorizationProperties.validate(properties)); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertDoesNotThrow(() -> chainedAuthzProperties.validate()); } @Test - void testChainTwoPlugins() { + void testChainedTwoPlugins() { Map properties = new HashMap<>(); - properties.put(HiveConstants.METASTORE_URIS, "thrift://localhost:9083"); + properties.put(METASTORE_URIS, "thrift://localhost:9083"); properties.put("gravitino.bypass.hive.metastore.client.capability.check", "true"); properties.put(IMPERSONATION_ENABLE, "true"); properties.put(AUTHORIZATION_PROVIDER, "chain"); @@ -65,7 +68,26 @@ void testChainTwoPlugins() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); - Assertions.assertDoesNotThrow(() -> ChainAuthorizationProperties.validate(properties)); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertDoesNotThrow(() -> chainedAuthzProperties.validate()); + } + + @Test + void testWithoutPlugins() { + Map properties = Maps.newHashMap(); + properties.put("authorization.chain.plugins", ""); + properties.put("authorization.chain.hive1.provider", "ranger"); + properties.put("authorization.chain.hive1.ranger.auth.type", "simple"); + properties.put("authorization.chain.hive1.ranger.admin.url", "http://localhost:6080"); + properties.put("authorization.chain.hive1.ranger.username", "admin"); + properties.put("authorization.chain.hive1.ranger.password", "admin"); + properties.put("authorization.chain.hive1.ranger.service.type", "hive"); + properties.put("authorization.chain.hive1.ranger.service.name", "hiveDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertThrows( + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -86,7 +108,9 @@ void testPluginsHasSpace() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); - Assertions.assertDoesNotThrow(() -> ChainAuthorizationProperties.validate(properties)); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); + Assertions.assertDoesNotThrow(() -> chainedAuthzProperties.validate()); } @Test @@ -107,8 +131,10 @@ void testPluginsOneButHasTowPluginConfig() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -129,8 +155,10 @@ void testPluginsHasPoint() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -151,8 +179,10 @@ void testErrorPluginName() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.plug3.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test @@ -173,14 +203,16 @@ void testDuplicationPluginName() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); Assertions.assertThrows( - IllegalArgumentException.class, () -> ChainAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, () -> chainedAuthzProperties.validate()); } @Test void testFetchRangerPrpoerties() { Map properties = new HashMap<>(); - properties.put(HiveConstants.METASTORE_URIS, "thrift://localhost:9083"); + properties.put(METASTORE_URIS, "thrift://localhost:9083"); properties.put("gravitino.bypass.hive.metastore.client.capability.check", "true"); properties.put(IMPERSONATION_ENABLE, "true"); properties.put(AUTHORIZATION_PROVIDER, "chain"); @@ -199,15 +231,25 @@ void testFetchRangerPrpoerties() { properties.put("authorization.chain.hdfs1.ranger.password", "admin"); properties.put("authorization.chain.hdfs1.ranger.service.type", "hadoop"); properties.put("authorization.chain.hdfs1.ranger.service.name", "hdfsDev"); + ChainedAuthorizationProperties chainedAuthzProperties = + new ChainedAuthorizationProperties(properties); - Map rangerHiveProperties = - ChainAuthorizationProperties.fetchAuthPluginProperties("hive1", properties); Assertions.assertDoesNotThrow( - () -> RangerAuthorizationProperties.validate(rangerHiveProperties)); + () -> { + Map rangerHiveProperties = + chainedAuthzProperties.fetchAuthPluginProperties("hive1"); + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(rangerHiveProperties); + rangerAuthProperties.validate(); + }); - Map rangerHDFSProperties = - ChainAuthorizationProperties.fetchAuthPluginProperties("hdfs1", properties); Assertions.assertDoesNotThrow( - () -> RangerAuthorizationProperties.validate(rangerHDFSProperties)); + () -> { + Map rangerHDFSProperties = + chainedAuthzProperties.fetchAuthPluginProperties("hdfs1"); + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(rangerHDFSProperties); + rangerAuthProperties.validate(); + }); } } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationProperties.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestRangerAuthorizationProperties.java similarity index 72% rename from authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationProperties.java rename to authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestRangerAuthorizationProperties.java index a90b164a21f..b2fcf6fc811 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationProperties.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/common/TestRangerAuthorizationProperties.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.ranger; +package org.apache.gravitino.authorization.common; import com.google.common.collect.Maps; import java.util.Map; @@ -33,7 +33,12 @@ void testRangerProperties() { properties.put("authorization.ranger.password", "admin"); properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); - Assertions.assertDoesNotThrow(() -> RangerAuthorizationProperties.validate(properties)); + Assertions.assertDoesNotThrow( + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -45,7 +50,12 @@ void testRangerPropertiesLoseAuthType() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -57,7 +67,12 @@ void testRangerPropertiesLoseAdminUrl() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -69,7 +84,12 @@ void testRangerPropertiesLoseUserName() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -81,7 +101,12 @@ void testRangerPropertiesLosePassword() { properties.put("authorization.ranger.service.type", "hive"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -93,7 +118,12 @@ void testRangerPropertiesLoseServiceType() { properties.put("authorization.ranger.password", "admin"); properties.put("authorization.ranger.service.name", "hiveDev"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } @Test @@ -105,6 +135,11 @@ void testRangerPropertiesLoseServiceName() { properties.put("authorization.ranger.password", "admin"); properties.put("authorization.ranger.service.type", "hive"); Assertions.assertThrows( - IllegalArgumentException.class, () -> RangerAuthorizationProperties.validate(properties)); + IllegalArgumentException.class, + () -> { + RangerAuthorizationProperties rangerAuthProperties = + new RangerAuthorizationProperties(properties); + rangerAuthProperties.validate(); + }); } } diff --git a/authorizations/authorization-jdbc/build.gradle.kts b/authorizations/authorization-jdbc/build.gradle.kts index 8b105908c26..1a61f7c0cf9 100644 --- a/authorizations/authorization-jdbc/build.gradle.kts +++ b/authorizations/authorization-jdbc/build.gradle.kts @@ -31,7 +31,9 @@ dependencies { implementation(project(":core")) { exclude(group = "*") } - + implementation(project(":authorizations:authorization-common")) { + exclude(group = "*") + } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) implementation(libs.guava) diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java index f889cee2240..d9bc28636c3 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java +++ b/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -40,6 +40,7 @@ import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.meta.AuditInfo; @@ -65,7 +66,8 @@ abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAutho public JdbcAuthorizationPlugin(Map config) { // Initialize the data source dataSource = new BasicDataSource(); - JdbcAuthorizationProperties.validate(config); + JdbcAuthorizationProperties jdbcAuthProperties = new JdbcAuthorizationProperties(config); + jdbcAuthProperties.validate(); String jdbcUrl = config.get(JdbcAuthorizationProperties.JDBC_URL); dataSource.setUrl(jdbcUrl); diff --git a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java b/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java index b72392a6cd8..e261fad78d2 100644 --- a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java +++ b/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java @@ -34,6 +34,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; import org.apache.gravitino.meta.RoleEntity; diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index a335e492b31..d410b1ee8d4 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -38,7 +38,12 @@ dependencies { implementation(project(":core")) { exclude(group = "*") } - + implementation(project(":catalogs:catalog-common")) { + exclude(group = "*") + } + implementation(project(":authorizations:authorization-common")) { + exclude(group = "*") + } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) implementation(libs.guava) @@ -47,10 +52,8 @@ dependencies { } implementation(libs.javax.ws.rs.api) implementation(libs.jettison) - compileOnly(libs.lombok) implementation(libs.mail) implementation(libs.ranger.intg) { - exclude("org.apache.hadoop", "hadoop-common") exclude("org.apache.hive", "hive-storage-api") exclude("org.apache.lucene") exclude("org.apache.solr") @@ -66,16 +69,15 @@ dependencies { exclude("org.eclipse.jetty") } implementation(libs.rome) - + compileOnly(libs.lombok) + testRuntimeOnly(libs.junit.jupiter.engine) testImplementation(project(":common")) testImplementation(project(":clients:client-java")) testImplementation(project(":server")) - testImplementation(project(":catalogs:catalog-common")) testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.testcontainers) - testRuntimeOnly(libs.junit.jupiter.engine) testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) testImplementation(libs.postgresql.driver) @@ -143,3 +145,16 @@ tasks.test { dependsOn(tasks.jar) } } + +val testJar by tasks.registering(Jar::class) { + archiveClassifier.set("tests") + from(sourceSets["test"].output) +} + +configurations { + create("testArtifacts") +} + +artifacts { + add("testArtifacts", testJar) +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java index 6aae714a359..b179b94c025 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorization.java @@ -18,10 +18,9 @@ */ package org.apache.gravitino.authorization.ranger; -import static org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties.RANGER_SERVICE_TYPE; - import com.google.common.base.Preconditions; import java.util.Map; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; @@ -36,9 +35,10 @@ public String shortName() { public AuthorizationPlugin newPlugin( String metalake, String catalogProvider, Map properties) { Preconditions.checkArgument( - properties.containsKey(RANGER_SERVICE_TYPE), - String.format("%s is required", RANGER_SERVICE_TYPE)); - String serviceType = properties.get(RANGER_SERVICE_TYPE).toUpperCase(); + properties.containsKey(RangerAuthorizationProperties.RANGER_SERVICE_TYPE), + String.format("%s is required", RangerAuthorizationProperties.RANGER_SERVICE_TYPE)); + String serviceType = + properties.get(RangerAuthorizationProperties.RANGER_SERVICE_TYPE).toUpperCase(); switch (serviceType) { case "HADOOPSQL": return new RangerAuthorizationHadoopSQLPlugin(metalake, properties); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java index 9afa77880e9..bc3d309e1d1 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.authorization.ranger; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -30,26 +31,29 @@ import java.util.Objects; import java.util.Set; import java.util.regex.Pattern; +import org.apache.gravitino.Catalog; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.Schema; import org.apache.gravitino.authorization.AuthorizationMetadataObject; import org.apache.gravitino.authorization.AuthorizationPrivilege; import org.apache.gravitino.authorization.AuthorizationSecurableObject; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.common.PathBasedMetadataObject; +import org.apache.gravitino.authorization.common.PathBasedSecurableObject; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.catalog.hive.HiveConstants; import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.file.Fileset; import org.apache.ranger.plugin.model.RangerPolicy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class RangerAuthorizationHDFSPlugin extends RangerAuthorizationPlugin { - private static final Logger LOG = LoggerFactory.getLogger(RangerAuthorizationHDFSPlugin.class); - private static final Pattern pattern = Pattern.compile("^hdfs://[^/]*"); public RangerAuthorizationHDFSPlugin(String metalake, Map config) { @@ -59,6 +63,38 @@ public RangerAuthorizationHDFSPlugin(String metalake, Map config @Override public Map> privilegesMappingRule() { return ImmutableMap.of( + Privilege.Name.USE_CATALOG, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.CREATE_CATALOG, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.USE_SCHEMA, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.CREATE_SCHEMA, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.CREATE_TABLE, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.MODIFY_TABLE, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.WRITE, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), + Privilege.Name.SELECT_TABLE, + ImmutableSet.of( + RangerPrivileges.RangerHdfsPrivilege.READ, + RangerPrivileges.RangerHdfsPrivilege.EXECUTE), Privilege.Name.READ_FILESET, ImmutableSet.of( RangerPrivileges.RangerHdfsPrivilege.READ, @@ -99,9 +135,9 @@ public AuthorizationSecurableObject generateAuthorizationSecurableObject( AuthorizationMetadataObject.Type type, Set privileges) { AuthorizationMetadataObject authMetadataObject = - new RangerPathBaseMetadataObject(AuthorizationMetadataObject.getLastName(names), type); + new PathBasedMetadataObject(AuthorizationMetadataObject.getLastName(names), type); authMetadataObject.validateAuthorizationMetadataObject(); - return new RangerPathBaseSecurableObject( + return new PathBasedSecurableObject( authMetadataObject.name(), authMetadataObject.type(), privileges); } @@ -137,10 +173,52 @@ public List translatePrivilege(SecurableObject sec .forEach( rangerPrivilege -> rangerPrivileges.add( - new RangerPrivileges.RangerHivePrivilegeImpl( + new RangerPrivileges.RangerHDFSPrivilegeImpl( rangerPrivilege, gravitinoPrivilege.condition()))); - switch (gravitinoPrivilege.name()) { + case USE_CATALOG: + case CREATE_CATALOG: + // When HDFS is used as the Hive storage layer, Hive does not support the + // `USE_CATALOG` and `CREATE_CATALOG` privileges. So, we ignore these + // in the RangerAuthorizationHDFSPlugin. + break; + case USE_SCHEMA: + break; + case CREATE_SCHEMA: + switch (securableObject.type()) { + case METALAKE: + case CATALOG: + { + String locationPath = getLocationPath(securableObject); + if (locationPath != null && !locationPath.isEmpty()) { + PathBasedMetadataObject rangerPathBaseMetadataObject = + new PathBasedMetadataObject( + locationPath, PathBasedMetadataObject.Type.PATH); + rangerSecurableObjects.add( + generateAuthorizationSecurableObject( + rangerPathBaseMetadataObject.names(), + PathBasedMetadataObject.Type.PATH, + rangerPrivileges)); + } + } + break; + case FILESET: + rangerSecurableObjects.add( + generateAuthorizationSecurableObject( + translateMetadataObject(securableObject).names(), + PathBasedMetadataObject.Type.PATH, + rangerPrivileges)); + break; + default: + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); + } + break; + case SELECT_TABLE: + case CREATE_TABLE: + case MODIFY_TABLE: + break; case CREATE_FILESET: // Ignore the Gravitino privilege `CREATE_FILESET` in the // RangerAuthorizationHDFSPlugin @@ -156,7 +234,7 @@ public List translatePrivilege(SecurableObject sec rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, rangerPrivileges)); break; default: @@ -166,10 +244,9 @@ public List translatePrivilege(SecurableObject sec } break; default: - LOG.warn( - "RangerAuthorizationHDFSPlugin -> privilege {} is not supported for the securable object: {}", - gravitinoPrivilege.name(), - securableObject.type()); + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); } }); @@ -183,12 +260,12 @@ public List translateOwner(MetadataObject gravitin case METALAKE: case CATALOG: case SCHEMA: - return rangerSecurableObjects; + break; case FILESET: rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, ownerMappingRule())); break; default: @@ -212,27 +289,77 @@ public AuthorizationMetadataObject translateMetadataObject(MetadataObject metada Preconditions.checkArgument( nsMetadataObject.size() > 0, "The metadata object must have at least one name."); - if (metadataObject.type() == MetadataObject.Type.FILESET) { - RangerPathBaseMetadataObject rangerHDFSMetadataObject = - new RangerPathBaseMetadataObject( - getFileSetPath(metadataObject), RangerPathBaseMetadataObject.Type.PATH); - rangerHDFSMetadataObject.validateAuthorizationMetadataObject(); - return rangerHDFSMetadataObject; - } else { - return new RangerPathBaseMetadataObject("", RangerPathBaseMetadataObject.Type.PATH); + PathBasedMetadataObject rangerPathBaseMetadataObject; + switch (metadataObject.type()) { + case METALAKE: + case CATALOG: + rangerPathBaseMetadataObject = + new PathBasedMetadataObject("", PathBasedMetadataObject.Type.PATH); + break; + case SCHEMA: + rangerPathBaseMetadataObject = + new PathBasedMetadataObject( + metadataObject.fullName(), PathBasedMetadataObject.Type.PATH); + break; + case FILESET: + rangerPathBaseMetadataObject = + new PathBasedMetadataObject( + getLocationPath(metadataObject), PathBasedMetadataObject.Type.PATH); + break; + default: + throw new AuthorizationPluginException( + "The metadata object type %s is not supported in the RangerAuthorizationHDFSPlugin", + metadataObject.type()); } + rangerPathBaseMetadataObject.validateAuthorizationMetadataObject(); + return rangerPathBaseMetadataObject; } - public String getFileSetPath(MetadataObject metadataObject) { - FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - NameIdentifier identifier = - NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); - Fileset fileset = filesetDispatcher.loadFileset(identifier); - Preconditions.checkArgument( - fileset != null, String.format("Fileset %s is not found", identifier)); - String filesetLocation = fileset.storageLocation(); - Preconditions.checkArgument( - filesetLocation != null, String.format("Fileset %s location is not found", identifier)); - return pattern.matcher(filesetLocation).replaceAll(""); + private NameIdentifier getObjectNameIdentifier(MetadataObject metadataObject) { + return NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); + } + + @VisibleForTesting + public String getLocationPath(MetadataObject metadataObject) throws NoSuchEntityException { + String locationPath = null; + switch (metadataObject.type()) { + case METALAKE: + case SCHEMA: + case TABLE: + break; + case CATALOG: + { + Namespace nsMetadataObj = Namespace.fromString(metadataObject.fullName()); + NameIdentifier ident = NameIdentifier.of(metalake, nsMetadataObj.level(0)); + Catalog catalog = GravitinoEnv.getInstance().catalogDispatcher().loadCatalog(ident); + if (catalog.provider().equals("hive")) { + Schema schema = + GravitinoEnv.getInstance() + .schemaDispatcher() + .loadSchema( + NameIdentifier.of( + metalake, nsMetadataObj.level(0), "default" /*Hive default schema*/)); + String defaultSchemaLocation = schema.properties().get(HiveConstants.LOCATION); + locationPath = pattern.matcher(defaultSchemaLocation).replaceAll(""); + } + } + break; + case FILESET: + FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); + NameIdentifier identifier = getObjectNameIdentifier(metadataObject); + Fileset fileset = filesetDispatcher.loadFileset(identifier); + Preconditions.checkArgument( + fileset != null, String.format("Fileset %s is not found", identifier)); + String filesetLocation = fileset.storageLocation(); + Preconditions.checkArgument( + filesetLocation != null, String.format("Fileset %s location is not found", identifier)); + locationPath = pattern.matcher(filesetLocation).replaceAll(""); + break; + default: + throw new AuthorizationPluginException( + "The metadata object type %s is not supported in the RangerAuthorizationHDFSPlugin", + metadataObject.type()); + } + return locationPath; } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java index b8e078d086e..aab19d31f36 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java @@ -154,25 +154,25 @@ public Set allowMetadataObjectTypesRule() { /** Translate the Gravitino securable object to the Ranger owner securable object. */ @Override public List translateOwner(MetadataObject gravitinoMetadataObject) { - List AuthorizationSecurableObjects = new ArrayList<>(); + List rangerSecurableObjects = new ArrayList<>(); switch (gravitinoMetadataObject.type()) { case METALAKE: case CATALOG: // Add `*` for the SCHEMA permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, ownerMappingRule())); // Add `*.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `*.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( RangerHelper.RESOURCE_ALL, @@ -183,20 +183,20 @@ public List translateOwner(MetadataObject gravitin break; case SCHEMA: // Add `{schema}` for the SCHEMA permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(gravitinoMetadataObject.name() /*Schema name*/), RangerHadoopSQLMetadataObject.Type.SCHEMA, ownerMappingRule())); // Add `{schema}.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( gravitinoMetadataObject.name() /*Schema name*/, RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `{schema}.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( gravitinoMetadataObject.name() /*Schema name*/, @@ -207,13 +207,13 @@ public List translateOwner(MetadataObject gravitin break; case TABLE: // Add `{schema}.{table}` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(gravitinoMetadataObject).names(), RangerHadoopSQLMetadataObject.Type.TABLE, ownerMappingRule())); // Add `{schema}.{table}.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( Stream.concat( translateMetadataObject(gravitinoMetadataObject).names().stream(), @@ -228,13 +228,13 @@ public List translateOwner(MetadataObject gravitin gravitinoMetadataObject.type()); } - return AuthorizationSecurableObjects; + return rangerSecurableObjects; } /** Translate the Gravitino securable object to the Ranger securable object. */ @Override public List translatePrivilege(SecurableObject securableObject) { - List AuthorizationSecurableObjects = new ArrayList<>(); + List rangerSecurableObjects = new ArrayList<>(); securableObject.privileges().stream() .filter(Objects::nonNull) @@ -262,7 +262,7 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add Ranger privilege(`SELECT`) to SCHEMA(`*`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -279,7 +279,7 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add Ranger privilege(`CREATE`) to SCHEMA(`*`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -296,7 +296,7 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add Ranger privilege(`SELECT`) to SCHEMA(`*`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -304,7 +304,7 @@ public List translatePrivilege(SecurableObject sec break; case SCHEMA: // Add Ranger privilege(`SELECT`) to SCHEMA(`{schema}`) - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of(securableObject.name() /*Schema name*/), RangerHadoopSQLMetadataObject.Type.SCHEMA, @@ -323,14 +323,14 @@ public List translatePrivilege(SecurableObject sec case METALAKE: case CATALOG: // Add `*.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `*.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( RangerHelper.RESOURCE_ALL, @@ -341,7 +341,7 @@ public List translatePrivilege(SecurableObject sec break; case SCHEMA: // Add `{schema}.*` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( securableObject.name() /*Schema name*/, @@ -349,7 +349,7 @@ public List translatePrivilege(SecurableObject sec RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `{schema}.*.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( ImmutableList.of( securableObject.name() /*Schema name*/, @@ -365,13 +365,13 @@ public List translatePrivilege(SecurableObject sec gravitinoPrivilege.name(), securableObject.type()); } else { // Add `{schema}.{table}` for the TABLE permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( translateMetadataObject(securableObject).names(), RangerHadoopSQLMetadataObject.Type.TABLE, rangerPrivileges)); // Add `{schema}.{table}.*` for the COLUMN permission - AuthorizationSecurableObjects.add( + rangerSecurableObjects.add( generateAuthorizationSecurableObject( Stream.concat( translateMetadataObject(securableObject).names().stream(), @@ -396,7 +396,7 @@ public List translatePrivilege(SecurableObject sec } }); - return AuthorizationSecurableObjects; + return rangerSecurableObjects; } /** diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 7a91ad54bf0..1198b68cb46 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -48,6 +48,7 @@ import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.reference.VXGroup; import org.apache.gravitino.authorization.ranger.reference.VXGroupList; import org.apache.gravitino.authorization.ranger.reference.VXUser; @@ -87,7 +88,9 @@ public abstract class RangerAuthorizationPlugin protected RangerAuthorizationPlugin(String metalake, Map config) { this.metalake = metalake; - RangerAuthorizationProperties.validate(config); + RangerAuthorizationProperties rangerAuthorizationProperties = + new RangerAuthorizationProperties(config); + rangerAuthorizationProperties.validate(); String rangerUrl = config.get(RangerAuthorizationProperties.RANGER_ADMIN_URL); String authType = config.get(RangerAuthorizationProperties.RANGER_AUTH_TYPE); rangerAdminName = config.get(RangerAuthorizationProperties.RANGER_USERNAME); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java index 8462a0e07a5..d64433b9feb 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHadoopSQLMetadataObject.java @@ -53,7 +53,7 @@ public static Type fromMetadataType(MetadataObject.Type metadataType) { } } throw new IllegalArgumentException( - "No matching RangerMetadataObject.Type for " + metadataType); + "No matching RangerHadoopSQLMetadataObject.Type for " + metadataType); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java index 4c2b2956c8c..64c454de61a 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java @@ -49,7 +49,7 @@ public class RangerHelper { private static final Logger LOG = LoggerFactory.getLogger(RangerHelper.class); public static final String MANAGED_BY_GRAVITINO = "MANAGED_BY_GRAVITINO"; - /** The `*` gives access to all resources */ + /** The `*` gives access to all table resources */ public static final String RESOURCE_ALL = "*"; /** The owner privileges, the owner can do anything on the metadata object */ private final Set ownerPrivileges; diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java index bbae16a6ba2..888d98f37d1 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java @@ -116,6 +116,32 @@ public boolean equalsTo(String value) { } } + public static class RangerHDFSPrivilegeImpl implements AuthorizationPrivilege { + private AuthorizationPrivilege rangerHDFSPrivilege; + private Privilege.Condition condition; + + public RangerHDFSPrivilegeImpl( + AuthorizationPrivilege rangerHivePrivilege, Privilege.Condition condition) { + this.rangerHDFSPrivilege = rangerHivePrivilege; + this.condition = condition; + } + + @Override + public String getName() { + return rangerHDFSPrivilege.getName(); + } + + @Override + public Privilege.Condition condition() { + return condition; + } + + @Override + public boolean equalsTo(String value) { + return rangerHDFSPrivilege.equalsTo(value); + } + } + static List>> allRangerPrivileges = Lists.newArrayList( RangerHadoopSQLPrivilege.class, RangerPrivileges.RangerHdfsPrivilege.class); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java index 4062263222b..4606fa68e70 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerAuthorizationHDFSPluginIT.java @@ -27,8 +27,8 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.common.PathBasedMetadataObject; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; -import org.apache.gravitino.authorization.ranger.RangerPathBaseMetadataObject; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -56,20 +56,19 @@ public void testTranslateMetadataObject() { MetadataObject metalake = MetadataObjects.parse(String.format("metalake1"), MetadataObject.Type.METALAKE); Assertions.assertEquals( - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(metalake).type()); MetadataObject catalog = MetadataObjects.parse(String.format("catalog1"), MetadataObject.Type.CATALOG); Assertions.assertEquals( - RangerPathBaseMetadataObject.Type.PATH, + PathBasedMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(catalog).type()); MetadataObject schema = MetadataObjects.parse(String.format("catalog1.schema1"), MetadataObject.Type.SCHEMA); Assertions.assertEquals( - RangerPathBaseMetadataObject.Type.PATH, - rangerAuthPlugin.translateMetadataObject(schema).type()); + PathBasedMetadataObject.Type.PATH, rangerAuthPlugin.translateMetadataObject(schema).type()); MetadataObject table = MetadataObjects.parse(String.format("catalog1.schema1.tab1"), MetadataObject.Type.TABLE); @@ -82,7 +81,7 @@ public void testTranslateMetadataObject() { AuthorizationMetadataObject rangerFileset = rangerAuthPlugin.translateMetadataObject(fileset); Assertions.assertEquals(1, rangerFileset.names().size()); Assertions.assertEquals("/test", rangerFileset.fullName()); - Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, rangerFileset.type()); + Assertions.assertEquals(PathBasedMetadataObject.Type.PATH, rangerFileset.type()); } @Test @@ -137,7 +136,7 @@ public void testTranslatePrivilege() { filesetInFileset1.forEach( securableObject -> { - Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, securableObject.type()); + Assertions.assertEquals(PathBasedMetadataObject.Type.PATH, securableObject.type()); Assertions.assertEquals("/test", securableObject.fullName()); Assertions.assertEquals(2, securableObject.privileges().size()); }); @@ -166,7 +165,7 @@ public void testTranslateOwner() { List filesetOwner = rangerAuthPlugin.translateOwner(fileset); Assertions.assertEquals(1, filesetOwner.size()); Assertions.assertEquals("/test", filesetOwner.get(0).fullName()); - Assertions.assertEquals(RangerPathBaseMetadataObject.Type.PATH, filesetOwner.get(0).type()); + Assertions.assertEquals(PathBasedMetadataObject.Type.PATH, filesetOwner.get(0).type()); Assertions.assertEquals(3, filesetOwner.get(0).privileges().size()); } } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java index 1fb9677d528..919551bd922 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerBaseE2EIT.java @@ -63,7 +63,6 @@ public abstract class RangerBaseE2EIT extends BaseIT { protected static GravitinoMetalake metalake; protected static Catalog catalog; protected static String HIVE_METASTORE_URIS; - protected static String RANGER_ADMIN_URL = null; protected static SparkSession sparkSession = null; protected static final String HADOOP_USER_NAME = "HADOOP_USER_NAME"; @@ -104,13 +103,13 @@ public abstract class RangerBaseE2EIT extends BaseIT { protected static final String SQL_DROP_TABLE = String.format("DROP TABLE %s", tableName); - protected static void generateRangerSparkSecurityXML() throws IOException { + protected static void generateRangerSparkSecurityXML(String modeName) throws IOException { String templatePath = String.join( File.separator, System.getenv("GRAVITINO_ROOT_DIR"), "authorizations", - "authorization-ranger", + modeName, "src", "test", "resources", @@ -120,7 +119,7 @@ protected static void generateRangerSparkSecurityXML() throws IOException { File.separator, System.getenv("GRAVITINO_ROOT_DIR"), "authorizations", - "authorization-ranger", + modeName, "build", "resources", "test", @@ -130,7 +129,7 @@ protected static void generateRangerSparkSecurityXML() throws IOException { FileUtils.readFileToString(new File(templatePath), StandardCharsets.UTF_8); templateContext = templateContext - .replace("__REPLACE__RANGER_ADMIN_URL", RANGER_ADMIN_URL) + .replace("__REPLACE__RANGER_ADMIN_URL", RangerITEnv.RANGER_ADMIN_URL) .replace("__REPLACE__RANGER_HIVE_REPO_NAME", RangerITEnv.RANGER_HIVE_REPO_NAME); FileUtils.writeStringToFile(new File(xmlPath), templateContext, StandardCharsets.UTF_8); } @@ -220,7 +219,7 @@ void testRenameMetalakeOrCatalog() { } @Test - protected void testCreateSchema() throws InterruptedException { + protected void testCreateSchema() throws InterruptedException, IOException { // Choose a catalog useCatalog(); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java index d8024afcc11..ab74b0449ae 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerFilesetIT.java @@ -45,7 +45,7 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.RangerHelper; import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.apache.gravitino.client.GravitinoMetalake; diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index 363f8f0b3a1..56cec3e9da3 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -20,7 +20,6 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; -import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -29,7 +28,7 @@ import org.apache.gravitino.Configs; import org.apache.gravitino.auth.AuthConstants; import org.apache.gravitino.auth.AuthenticatorType; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.catalog.hive.HiveConstants; import org.apache.gravitino.exceptions.UserAlreadyExistsException; import org.apache.gravitino.integration.test.container.HiveContainer; @@ -67,18 +66,13 @@ public void startIntegrationTest() throws Exception { RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); - RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); - HIVE_METASTORE_URIS = String.format( "thrift://%s:%d", containerSuite.getHiveRangerContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); - generateRangerSparkSecurityXML(); + generateRangerSparkSecurityXML("authorization-ranger"); sparkSession = SparkSession.builder() @@ -186,7 +180,7 @@ public void createCatalog() { RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME, RangerAuthorizationProperties.RANGER_ADMIN_URL, - RANGER_ADMIN_URL, + RangerITEnv.RANGER_ADMIN_URL, RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType, RangerAuthorizationProperties.RANGER_USERNAME, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 2efc1e9dd60..913482ef03e 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.authorization.ranger.integration.test; +import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import static org.mockito.Mockito.doReturn; import com.google.common.collect.ImmutableList; @@ -32,10 +33,10 @@ import org.apache.gravitino.authorization.AuthorizationSecurableObject; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.RangerAuthorizationHDFSPlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationHadoopSQLPlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; import org.apache.gravitino.authorization.ranger.RangerHelper; import org.apache.gravitino.authorization.ranger.RangerPrivileges; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; @@ -87,11 +88,15 @@ public class RangerITEnv { public static RangerAuthorizationPlugin rangerAuthHivePlugin; public static RangerAuthorizationPlugin rangerAuthHDFSPlugin; protected static RangerHelper rangerHelper; - protected static RangerHelper rangerHDFSHelper; + public static String RANGER_ADMIN_URL = null; public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { containerSuite.startRangerContainer(); + RANGER_ADMIN_URL = + String.format( + "http://%s:%d", + containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); rangerClient = containerSuite.getRangerContainer().rangerClient; rangerAuthHivePlugin = @@ -134,7 +139,7 @@ public static void init(String metalakeName, boolean allowAnyoneAccessHDFS) { "HDFS", RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HDFS_REPO_NAME))); - doReturn("/test").when(spyRangerAuthorizationHDFSPlugin).getFileSetPath(Mockito.any()); + doReturn("/test").when(spyRangerAuthorizationHDFSPlugin).getLocationPath(Mockito.any()); rangerAuthHDFSPlugin = spyRangerAuthorizationHDFSPlugin; rangerHelper = diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java index 8f6f769504a..3e3d0d24234 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerIcebergE2EIT.java @@ -21,7 +21,6 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; import static org.apache.gravitino.catalog.hive.HiveConstants.IMPERSONATION_ENABLE; -import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -35,7 +34,7 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.RangerContainer; @@ -70,18 +69,13 @@ public void startIntegrationTest() throws Exception { RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); - RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); - HIVE_METASTORE_URIS = String.format( "thrift://%s:%d", containerSuite.getHiveRangerContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); - generateRangerSparkSecurityXML(); + generateRangerSparkSecurityXML("authorization-ranger"); sparkSession = SparkSession.builder() @@ -179,7 +173,7 @@ public void createCatalog() { properties.put(RangerAuthorizationProperties.RANGER_SERVICE_TYPE, "HadoopSQL"); properties.put( RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME); - properties.put(RangerAuthorizationProperties.RANGER_ADMIN_URL, RANGER_ADMIN_URL); + properties.put(RangerAuthorizationProperties.RANGER_ADMIN_URL, RangerITEnv.RANGER_ADMIN_URL); properties.put(RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType); properties.put(RangerAuthorizationProperties.RANGER_USERNAME, RangerContainer.rangerUserName); properties.put(RangerAuthorizationProperties.RANGER_PASSWORD, RangerContainer.rangerPassword); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java index 2773610048e..c37fd20c85f 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerPaimonE2EIT.java @@ -20,7 +20,6 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; -import static org.apache.gravitino.integration.test.container.RangerContainer.RANGER_SERVER_PORT; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -34,7 +33,7 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationProperties; +import org.apache.gravitino.authorization.common.RangerAuthorizationProperties; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.RangerContainer; import org.apache.gravitino.integration.test.util.GravitinoITUtils; @@ -69,18 +68,13 @@ public void startIntegrationTest() throws Exception { RangerITEnv.init(RangerBaseE2EIT.metalakeName, true); RangerITEnv.startHiveRangerContainer(); - RANGER_ADMIN_URL = - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), RANGER_SERVER_PORT); - HIVE_METASTORE_URIS = String.format( "thrift://%s:%d", containerSuite.getHiveRangerContainer().getContainerIpAddress(), HiveContainer.HIVE_METASTORE_PORT); - generateRangerSparkSecurityXML(); + generateRangerSparkSecurityXML("authorization-ranger"); sparkSession = SparkSession.builder() @@ -199,7 +193,7 @@ public void createCatalog() { RangerAuthorizationProperties.RANGER_SERVICE_NAME, RangerITEnv.RANGER_HIVE_REPO_NAME, RangerAuthorizationProperties.RANGER_ADMIN_URL, - RANGER_ADMIN_URL, + RangerITEnv.RANGER_ADMIN_URL, RangerAuthorizationProperties.RANGER_AUTH_TYPE, RangerContainer.authType, RangerAuthorizationProperties.RANGER_USERNAME, diff --git a/authorizations/build.gradle.kts b/authorizations/build.gradle.kts index 043fbfec673..354b36aae64 100644 --- a/authorizations/build.gradle.kts +++ b/authorizations/build.gradle.kts @@ -17,6 +17,18 @@ * under the License. */ -tasks.all { - enabled = false -} \ No newline at end of file +tasks { + test { + subprojects.forEach { + dependsOn(":${project.name}:${it.name}:test") + } + } + + register("copyLibAndConfig", Copy::class) { + subprojects.forEach { + if (!it.name.startsWith("authorization-common")) { + dependsOn(":${project.name}:${it.name}:copyLibAndConfig") + } + } + } +} diff --git a/build.gradle.kts b/build.gradle.kts index 5e93992e34e..c64997f3a90 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -583,7 +583,7 @@ tasks { val outputDir = projectDir.dir("distribution") val compileDistribution by registering { - dependsOn(":web:web:build", "copySubprojectDependencies", "copyCatalogLibAndConfigs", "copyAuthorizationLibAndConfigs", "copySubprojectLib", "iceberg:iceberg-rest-server:copyLibAndConfigs") + dependsOn(":web:web:build", "copySubprojectDependencies", "copyCatalogLibAndConfigs", ":authorizations:copyLibAndConfig", "copySubprojectLib", "iceberg:iceberg-rest-server:copyLibAndConfigs") group = "gravitino distribution" outputs.dir(projectDir.dir("distribution/package")) @@ -829,12 +829,6 @@ tasks { ) } - register("copyAuthorizationLibAndConfigs", Copy::class) { - dependsOn( - ":authorizations:authorization-ranger:copyLibAndConfig" - ) - } - clean { dependsOn(cleanDistribution) } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java index 3d71948b744..36307f3ba4b 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import java.lang.reflect.Field; import java.time.LocalDate; import java.util.Collections; import java.util.Map; @@ -423,21 +422,4 @@ private static void loadCatalogWithAnotherClient() { anotherCatalogWithNotExistingName = anotherClientWithNotExistingName.loadMetalake(METALAKE_NAME).loadCatalog(CATALOG_NAME); } - - public static void setEnv(String key, String value) { - try { - Map env = System.getenv(); - Class cl = env.getClass(); - Field field = cl.getDeclaredField("m"); - field.setAccessible(true); - Map writableEnv = (Map) field.get(env); - if (value == null) { - writableEnv.remove(key); - } else { - writableEnv.put(key, value); - } - } catch (Exception e) { - throw new IllegalStateException("Failed to set environment variable", e); - } - } } diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index 43bc74bb2a1..4a46952f87e 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -58,7 +58,6 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Catalog; import org.apache.gravitino.CatalogChange; @@ -79,6 +78,7 @@ import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.PropertyEntry; import org.apache.gravitino.connector.SupportsSchemas; +import org.apache.gravitino.connector.authorization.BaseAuthorization; import org.apache.gravitino.connector.capability.Capability; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; import org.apache.gravitino.exceptions.CatalogInUseException; @@ -944,7 +944,7 @@ private IsolatedClassLoader createClassLoader(String provider, Map libAndResourcesPaths = Lists.newArrayList(catalogPkgPath, catalogConfPath); - buildAuthorizationPkgPath(conf).ifPresent(libAndResourcesPaths::add); + BaseAuthorization.buildAuthorizationPkgPath(conf).ifPresent(libAndResourcesPaths::add); return IsolatedClassLoader.buildClassLoader(libAndResourcesPaths); } else { // This will use the current class loader, it is mainly used for test. @@ -1061,37 +1061,6 @@ private String buildPkgPath(Map conf, String provider) { return pkgPath; } - private Optional buildAuthorizationPkgPath(Map conf) { - String gravitinoHome = System.getenv("GRAVITINO_HOME"); - Preconditions.checkArgument(gravitinoHome != null, "GRAVITINO_HOME not set"); - boolean testEnv = System.getenv("GRAVITINO_TEST") != null; - - String authorizationProvider = conf.get(Catalog.AUTHORIZATION_PROVIDER); - if (StringUtils.isBlank(authorizationProvider)) { - return Optional.empty(); - } - - String pkgPath; - if (testEnv) { - // In test, the authorization package is under the build directory. - pkgPath = - String.join( - File.separator, - gravitinoHome, - "authorizations", - "authorization-" + authorizationProvider, - "build", - "libs"); - } else { - // In real environment, the authorization package is under the authorization directory. - pkgPath = - String.join( - File.separator, gravitinoHome, "authorizations", authorizationProvider, "libs"); - } - - return Optional.of(pkgPath); - } - private Class lookupCatalogProvider(String provider, ClassLoader cl) { ServiceLoader loader = ServiceLoader.load(CatalogProvider.class, cl); diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index 88fd47ab998..218c2a428b3 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -19,22 +19,16 @@ package org.apache.gravitino.connector; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import com.google.common.collect.Streams; import java.io.Closeable; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.ServiceLoader; -import java.util.stream.Collectors; import org.apache.gravitino.Audit; import org.apache.gravitino.Catalog; import org.apache.gravitino.CatalogProvider; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; -import org.apache.gravitino.connector.authorization.AuthorizationProvider; import org.apache.gravitino.connector.authorization.BaseAuthorization; import org.apache.gravitino.connector.capability.Capability; import org.apache.gravitino.meta.CatalogEntity; @@ -209,34 +203,7 @@ public void initAuthorizationPluginInstance(IsolatedClassLoader classLoader) { } try { BaseAuthorization authorization = - classLoader.withClassLoader( - cl -> { - try { - ServiceLoader loader = - ServiceLoader.load(AuthorizationProvider.class, cl); - - List> providers = - Streams.stream(loader.iterator()) - .filter(p -> p.shortName().equalsIgnoreCase(authorizationProvider)) - .map(AuthorizationProvider::getClass) - .collect(Collectors.toList()); - if (providers.isEmpty()) { - throw new IllegalArgumentException( - "No authorization provider found for: " + authorizationProvider); - } else if (providers.size() > 1) { - throw new IllegalArgumentException( - "Multiple authorization providers found for: " - + authorizationProvider); - } - return (BaseAuthorization) - Iterables.getOnlyElement(providers) - .getDeclaredConstructor() - .newInstance(); - } catch (Exception e) { - LOG.error("Failed to create authorization instance", e); - throw new RuntimeException(e); - } - }); + BaseAuthorization.createAuthorization(classLoader, authorizationProvider); authorizationPlugin = authorization.newPlugin(entity.namespace().level(0), provider(), this.conf); } catch (Exception e) { diff --git a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java index 173ad3527a8..740f808e4df 100644 --- a/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java +++ b/core/src/main/java/org/apache/gravitino/connector/authorization/BaseAuthorization.java @@ -18,9 +18,20 @@ */ package org.apache.gravitino.connector.authorization; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.common.collect.Streams; import java.io.Closeable; +import java.io.File; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.utils.IsolatedClassLoader; /** * The abstract base class for Authorization implementations.
@@ -46,4 +57,65 @@ public abstract AuthorizationPlugin newPlugin( @Override public void close() throws IOException {} + + public static BaseAuthorization createAuthorization( + IsolatedClassLoader classLoader, String authorizationProvider) throws Exception { + BaseAuthorization authorization = + classLoader.withClassLoader( + cl -> { + try { + ServiceLoader loader = + ServiceLoader.load(AuthorizationProvider.class, cl); + + List> providers = + Streams.stream(loader.iterator()) + .filter(p -> p.shortName().equalsIgnoreCase(authorizationProvider)) + .map(AuthorizationProvider::getClass) + .collect(Collectors.toList()); + if (providers.isEmpty()) { + throw new IllegalArgumentException( + "No authorization provider found for: " + authorizationProvider); + } else if (providers.size() > 1) { + throw new IllegalArgumentException( + "Multiple authorization providers found for: " + authorizationProvider); + } + return (BaseAuthorization) + Iterables.getOnlyElement(providers).getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return authorization; + } + + public static Optional buildAuthorizationPkgPath(Map conf) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Preconditions.checkArgument(gravitinoHome != null, "GRAVITINO_HOME not set"); + boolean testEnv = System.getenv("GRAVITINO_TEST") != null; + + String authorizationProvider = conf.get(Catalog.AUTHORIZATION_PROVIDER); + if (StringUtils.isBlank(authorizationProvider)) { + return Optional.empty(); + } + + String pkgPath; + if (testEnv) { + // In test, the authorization package is under the build directory. + pkgPath = + String.join( + File.separator, + gravitinoHome, + "authorizations", + "authorization-" + authorizationProvider, + "build", + "libs"); + } else { + // In real environment, the authorization package is under the authorization directory. + pkgPath = + String.join( + File.separator, gravitinoHome, "authorizations", authorizationProvider, "libs"); + } + + return Optional.of(pkgPath); + } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java index 554ef0cec8b..4ee37b4ddec 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/TestAuthorization.java @@ -24,8 +24,8 @@ import org.apache.gravitino.Catalog; import org.apache.gravitino.Namespace; import org.apache.gravitino.TestCatalog; -import org.apache.gravitino.connector.authorization.mysql.TestMySQLAuthorizationPlugin; -import org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorizationPlugin; +import org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorizationHDFSPlugin; +import org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorizationHadoopSQLPlugin; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.utils.IsolatedClassLoader; @@ -35,7 +35,7 @@ public class TestAuthorization { private static TestCatalog hiveCatalog; - private static TestCatalog mySQLCatalog; + private static TestCatalog filesetCatalog; @BeforeAll public static void setUp() throws Exception { @@ -54,49 +54,59 @@ public static void setUp() throws Exception { hiveCatalog = new TestCatalog() - .withCatalogConf(ImmutableMap.of(Catalog.AUTHORIZATION_PROVIDER, "ranger")) + .withCatalogConf( + ImmutableMap.of( + Catalog.AUTHORIZATION_PROVIDER, + "test-ranger", + "authorization.ranger.service.type", + "HadoopSQL")) .withCatalogEntity(hiveCatalogEntity); IsolatedClassLoader isolatedClassLoader = new IsolatedClassLoader( Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); hiveCatalog.initAuthorizationPluginInstance(isolatedClassLoader); - CatalogEntity mySQLEntity = + CatalogEntity filesetEntity = CatalogEntity.builder() .withId(2L) .withName("catalog-test2") .withNamespace(Namespace.of("default")) - .withType(Catalog.Type.RELATIONAL) + .withType(Catalog.Type.FILESET) .withProvider("test") .withAuditInfo(auditInfo) .build(); - mySQLCatalog = + filesetCatalog = new TestCatalog() - .withCatalogConf(ImmutableMap.of(Catalog.AUTHORIZATION_PROVIDER, "mysql")) - .withCatalogEntity(mySQLEntity); - mySQLCatalog.initAuthorizationPluginInstance(isolatedClassLoader); + .withCatalogConf( + ImmutableMap.of( + Catalog.AUTHORIZATION_PROVIDER, + "test-ranger", + "authorization.ranger.service.type", + "HDFS")) + .withCatalogEntity(filesetEntity); + filesetCatalog.initAuthorizationPluginInstance(isolatedClassLoader); } @Test - public void testRangerAuthorization() { - AuthorizationPlugin rangerAuthPlugin = hiveCatalog.getAuthorizationPlugin(); - Assertions.assertInstanceOf(TestRangerAuthorizationPlugin.class, rangerAuthPlugin); - TestRangerAuthorizationPlugin testRangerAuthPlugin = - (TestRangerAuthorizationPlugin) rangerAuthPlugin; - Assertions.assertFalse(testRangerAuthPlugin.callOnCreateRole1); - rangerAuthPlugin.onRoleCreated(null); - Assertions.assertTrue(testRangerAuthPlugin.callOnCreateRole1); + public void testRangerHadoopSQLAuthorization() { + AuthorizationPlugin rangerHiveAuthPlugin = hiveCatalog.getAuthorizationPlugin(); + Assertions.assertInstanceOf(TestRangerAuthorizationHadoopSQLPlugin.class, rangerHiveAuthPlugin); + TestRangerAuthorizationHadoopSQLPlugin testRangerAuthHadoopSQLPlugin = + (TestRangerAuthorizationHadoopSQLPlugin) rangerHiveAuthPlugin; + Assertions.assertFalse(testRangerAuthHadoopSQLPlugin.callOnCreateRole1); + rangerHiveAuthPlugin.onRoleCreated(null); + Assertions.assertTrue(testRangerAuthHadoopSQLPlugin.callOnCreateRole1); } @Test - public void testMySQLAuthorization() { - AuthorizationPlugin mySQLAuthPlugin = mySQLCatalog.getAuthorizationPlugin(); - Assertions.assertInstanceOf(TestMySQLAuthorizationPlugin.class, mySQLAuthPlugin); - TestMySQLAuthorizationPlugin testMySQLAuthPlugin = - (TestMySQLAuthorizationPlugin) mySQLAuthPlugin; - Assertions.assertFalse(testMySQLAuthPlugin.callOnCreateRole2); - mySQLAuthPlugin.onRoleCreated(null); - Assertions.assertTrue(testMySQLAuthPlugin.callOnCreateRole2); + public void testRangerHDFSAuthorization() { + AuthorizationPlugin rangerHDFSAuthPlugin = filesetCatalog.getAuthorizationPlugin(); + Assertions.assertInstanceOf(TestRangerAuthorizationHDFSPlugin.class, rangerHDFSAuthPlugin); + TestRangerAuthorizationHDFSPlugin testRangerAuthHDFSPlugin = + (TestRangerAuthorizationHDFSPlugin) rangerHDFSAuthPlugin; + Assertions.assertFalse(testRangerAuthHDFSPlugin.callOnCreateRole2); + rangerHDFSAuthPlugin.onRoleCreated(null); + Assertions.assertTrue(testRangerAuthHDFSPlugin.callOnCreateRole2); } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java index 9df9a8d63b7..1709c90319f 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorization.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.connector.authorization.ranger; +import com.google.common.base.Preconditions; import java.util.Map; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; @@ -28,12 +29,23 @@ public TestRangerAuthorization() {} @Override public String shortName() { - return "ranger"; + return "test-ranger"; } @Override public AuthorizationPlugin newPlugin( - String metalake, String catalogProvider, Map config) { - return new TestRangerAuthorizationPlugin(); + String metalake, String catalogProvider, Map properties) { + Preconditions.checkArgument( + properties.containsKey("authorization.ranger.service.type"), + String.format("%s is required", "authorization.ranger.service.type")); + String serviceType = properties.get("authorization.ranger.service.type").toUpperCase(); + switch (serviceType) { + case "HADOOPSQL": + return new TestRangerAuthorizationHadoopSQLPlugin(); + case "HDFS": + return new TestRangerAuthorizationHDFSPlugin(); + default: + throw new IllegalArgumentException("Unsupported service type: " + serviceType); + } } } diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHDFSPlugin.java similarity index 95% rename from core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java rename to core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHDFSPlugin.java index e078eda410e..fdc28f8143e 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/mysql/TestMySQLAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHDFSPlugin.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.connector.authorization.mysql; +package org.apache.gravitino.connector.authorization.ranger; import java.io.IOException; import java.util.List; @@ -29,7 +29,7 @@ import org.apache.gravitino.authorization.User; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; -public class TestMySQLAuthorizationPlugin implements AuthorizationPlugin { +public class TestRangerAuthorizationHDFSPlugin implements AuthorizationPlugin { public boolean callOnCreateRole2 = false; @Override diff --git a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHadoopSQLPlugin.java similarity index 97% rename from core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java rename to core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHadoopSQLPlugin.java index 8a68f825d0e..10dbe521e6c 100644 --- a/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationPlugin.java +++ b/core/src/test/java/org/apache/gravitino/connector/authorization/ranger/TestRangerAuthorizationHadoopSQLPlugin.java @@ -29,7 +29,7 @@ import org.apache.gravitino.authorization.User; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; -public class TestRangerAuthorizationPlugin implements AuthorizationPlugin { +public class TestRangerAuthorizationHadoopSQLPlugin implements AuthorizationPlugin { public boolean callOnCreateRole1 = false; @Override diff --git a/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider b/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider index e49cb8937e0..b7219fdc279 100644 --- a/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider +++ b/core/src/test/resources/META-INF/services/org.apache.gravitino.connector.authorization.AuthorizationProvider @@ -16,5 +16,4 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorization -org.apache.gravitino.connector.authorization.mysql.TestMySQLAuthorization \ No newline at end of file +org.apache.gravitino.connector.authorization.ranger.TestRangerAuthorization \ No newline at end of file diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index a25ad4cff8f..283169a76a9 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -54,7 +54,10 @@ dependencies { exclude("org.elasticsearch.client") exclude("org.elasticsearch.plugin") } - + testImplementation(libs.hadoop3.common) { + exclude("com.sun.jersey") + exclude("javax.servlet", "servlet-api") + } testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java index fcf8ebb2b9c..f8cbe508f87 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/BaseIT.java @@ -26,6 +26,7 @@ import com.google.common.base.Splitter; import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -57,6 +58,7 @@ import org.apache.gravitino.server.GravitinoServer; import org.apache.gravitino.server.ServerConfig; import org.apache.gravitino.server.web.JettyServerConfig; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; @@ -422,4 +424,40 @@ protected static void copyBundleJarsToHadoop(String bundleName) { String hadoopLibDirs = ITUtils.joinPath(gravitinoHome, "catalogs", "hadoop", "libs"); copyBundleJarsToDirectory(bundleName, hadoopLibDirs); } + + public static void runInEnv(String key, String value, Runnable lambda) { + String originalValue = System.getenv(key); + try { + setEnv(key, value); + if (key.equals("HADOOP_USER_NAME") && value != null) { + UserGroupInformation.setLoginUser(null); + System.setProperty("user.name", value); + } + lambda.run(); + } catch (Exception e) { + throw new IllegalStateException("Failed to set environment variable", e); + } finally { + setEnv(key, originalValue); + if (key.equals("HADOOP_USER_NAME") && value != null) { + System.setProperty("user.name", originalValue); + } + } + } + + public static void setEnv(String key, String value) { + try { + Map env = System.getenv(); + Class cl = env.getClass(); + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Map writableEnv = (Map) field.get(env); + if (value == null) { + writableEnv.remove(key); + } else { + writableEnv.put(key, value); + } + } catch (Exception e) { + throw new IllegalStateException("Failed to set environment variable", e); + } + } } diff --git a/settings.gradle.kts b/settings.gradle.kts index b3eb56578aa..f38443db206 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,7 @@ if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() == true) } include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") -include("authorizations:authorization-ranger", "authorizations:authorization-jdbc") +include("authorizations:authorization-ranger", "authorizations:authorization-jdbc", "authorizations:authorization-common", "authorizations:authorization-chain") include("trino-connector:trino-connector", "trino-connector:integration-test") include("spark-connector:spark-common") // kyuubi hive connector doesn't support 2.13 for Spark3.3 From c2d1b1ef9deff1e74bdb3a3f3c37f2b5a5c08a35 Mon Sep 17 00:00:00 2001 From: cai can <94670132+caican00@users.noreply.github.com> Date: Wed, 25 Dec 2024 15:49:32 +0800 Subject: [PATCH 2/7] [#4714] feat(paimon-spark-connector): Add tests for partitionManagement of paimon table in paimon spark connector (#5860) ### What changes were proposed in this pull request? Add tests for partitionManagement of paimon table in paimon spark connector ### Why are the changes needed? Fix: https://github.com/apache/gravitino/issues/4714 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? New ITs. --------- Co-authored-by: caican --- .../test/paimon/SparkPaimonCatalogIT.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java index c77a4642eec..9d036482857 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/paimon/SparkPaimonCatalogIT.java @@ -104,6 +104,37 @@ void testPaimonPartitions() { checkDirExists(partitionPath); } + @Test + void testPaimonPartitionManagement() { + testPaimonListAndDropPartition(); + // TODO: replace, add and load partition operations are unsupported now. + } + + private void testPaimonListAndDropPartition() { + String tableName = "test_paimon_drop_partition"; + dropTableIfExists(tableName); + String createTableSQL = getCreatePaimonSimpleTableString(tableName); + createTableSQL = createTableSQL + " PARTITIONED BY (name);"; + sql(createTableSQL); + + String insertData = + String.format( + "INSERT into %s values(1,'a','beijing'), (2,'b','beijing'), (3,'c','beijing');", + tableName); + sql(insertData); + List queryResult = getTableData(tableName); + Assertions.assertEquals(3, queryResult.size()); + + List partitions = getQueryData(String.format("show partitions %s", tableName)); + Assertions.assertEquals(3, partitions.size()); + Assertions.assertEquals("name=a;name=b;name=c", String.join(";", partitions)); + + sql(String.format("ALTER TABLE %s DROP PARTITION (`name`='a')", tableName)); + partitions = getQueryData(String.format("show partitions %s", tableName)); + Assertions.assertEquals(2, partitions.size()); + Assertions.assertEquals("name=b;name=c", String.join(";", partitions)); + } + private String getCreatePaimonSimpleTableString(String tableName) { return String.format( "CREATE TABLE %s (id INT COMMENT 'id comment', name STRING COMMENT '', address STRING COMMENT '') USING paimon", From 304ef70c39e2e6d45be8c055f2092610d36768b9 Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Thu, 26 Dec 2024 06:11:31 +0800 Subject: [PATCH 3/7] [#5861] improvement(CLI): Refactor the validation logic in the handle methods (#5972) ### What changes were proposed in this pull request? refactor the validation logic of all entities and add test case, just like validation of table command #5906 . A hint is provided when the user's output is missing the required arguments. for example: ```bash gcli column list -m demo_metalake, --name Hive_catalog # Malformed entity name. # Missing required argument(s): schema, table gcli column details -m demo_metalake, --name Hive_catalog --audit # Malformed entity name. # Missing required argument(s): schema, table, column gcli user delete -m demo_metalake Missing --user option. ``` Currently, the Role command needs to be refactored and opened as a separate issue ### Why are the changes needed? Fix: #5861 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? local test --- .../apache/gravitino/cli/ErrorMessages.java | 1 + .../gravitino/cli/GravitinoCommandLine.java | 87 +++++--- .../gravitino/cli/TestCatalogCommands.java | 33 +++ .../gravitino/cli/TestColumnCommands.java | 211 ++++++++++++++++++ .../gravitino/cli/TestFilesetCommands.java | 155 +++++++++++++ .../gravitino/cli/TestGroupCommands.java | 43 ++++ .../org/apache/gravitino/cli/TestMain.java | 1 - .../gravitino/cli/TestTableCommands.java | 76 ++++--- .../apache/gravitino/cli/TestTagCommands.java | 42 ++++ .../gravitino/cli/TestTopicCommands.java | 154 +++++++++++++ .../gravitino/cli/TestUserCommands.java | 42 ++++ 11 files changed, 778 insertions(+), 67 deletions(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java index 3423cee07f7..1d6db1a5acd 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ErrorMessages.java @@ -32,6 +32,7 @@ public class ErrorMessages { public static final String MISSING_NAME = "Missing --name option."; public static final String MISSING_GROUP = "Missing --group option."; public static final String MISSING_USER = "Missing --user option."; + public static final String MISSING_TAG = "Missing --tag option."; public static final String METALAKE_EXISTS = "Metalake already exists."; public static final String CATALOG_EXISTS = "Catalog already exists."; public static final String SCHEMA_EXISTS = "Schema already exists."; diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 7c8539ba1c7..48d97294350 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -31,8 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; @@ -255,6 +253,7 @@ private void handleCatalogCommand() { String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); // Handle the CommandActions.LIST action separately as it doesn't use `catalog` if (CommandActions.LIST.equals(command)) { @@ -263,6 +262,8 @@ private void handleCatalogCommand() { } String catalog = name.getCatalogName(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -343,29 +344,21 @@ private void handleSchemaCommand() { String catalog = name.getCatalogName(); Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); if (metalake == null) missingEntities.add(CommandEntities.METALAKE); if (catalog == null) missingEntities.add(CommandEntities.CATALOG); // Handle the CommandActions.LIST action separately as it doesn't use `schema` if (CommandActions.LIST.equals(command)) { - if (!missingEntities.isEmpty()) { - System.err.println("Missing required argument(s): " + COMMA_JOINER.join(missingEntities)); - Main.exit(-1); - } + checkEntities(missingEntities); newListSchema(url, ignore, metalake, catalog).handle(); return; } String schema = name.getSchemaName(); - if (schema == null) { - missingEntities.add(CommandEntities.SCHEMA); - } - - if (!missingEntities.isEmpty()) { - System.err.println("Missing required argument(s): " + COMMA_JOINER.join(missingEntities)); - Main.exit(-1); - } + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -421,33 +414,20 @@ private void handleTableCommand() { String schema = name.getSchemaName(); Command.setAuthenticationMode(auth, userName); - List missingEntities = - Stream.of( - catalog == null ? CommandEntities.CATALOG : null, - schema == null ? CommandEntities.SCHEMA : null) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); // Handle CommandActions.LIST action separately as it doesn't require the `table` if (CommandActions.LIST.equals(command)) { - if (!missingEntities.isEmpty()) { - System.err.println( - "Missing required argument(s): " + Joiner.on(", ").join(missingEntities)); - Main.exit(-1); - } + checkEntities(missingEntities); newListTables(url, ignore, metalake, catalog, schema).handle(); return; } String table = name.getTableName(); - if (table == null) { - missingEntities.add(CommandEntities.TABLE); - } - - if (!missingEntities.isEmpty()) { - System.err.println("Missing required argument(s): " + Joiner.on(", ").join(missingEntities)); - Main.exit(-1); - } + if (table == null) missingEntities.add(CommandEntities.TABLE); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -527,7 +507,7 @@ protected void handleUserCommand() { if (user == null && !CommandActions.LIST.equals(command)) { System.err.println(ErrorMessages.MISSING_USER); - return; + Main.exit(-1); } switch (command) { @@ -588,7 +568,7 @@ protected void handleGroupCommand() { if (group == null && !CommandActions.LIST.equals(command)) { System.err.println(ErrorMessages.MISSING_GROUP); - return; + Main.exit(-1); } switch (command) { @@ -647,6 +627,13 @@ protected void handleTagCommand() { Command.setAuthenticationMode(auth, userName); String[] tags = line.getOptionValues(GravitinoOptions.TAG); + if (tags == null + && !((CommandActions.REMOVE.equals(command) && line.hasOption(GravitinoOptions.FORCE)) + || CommandActions.LIST.equals(command))) { + System.err.println(ErrorMessages.MISSING_TAG); + Main.exit(-1); + } + if (tags != null) { tags = Arrays.stream(tags).distinct().toArray(String[]::new); } @@ -790,12 +777,20 @@ private void handleColumnCommand() { Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + if (table == null) missingEntities.add(CommandEntities.TABLE); + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); newListColumns(url, ignore, metalake, catalog, schema, table).handle(); return; } String column = name.getColumnName(); + if (column == null) missingEntities.add(CommandEntities.COLUMN); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -965,12 +960,19 @@ private void handleTopicCommand() { Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); newListTopics(url, ignore, metalake, catalog, schema).handle(); return; } String topic = name.getTopicName(); + if (topic == null) missingEntities.add(CommandEntities.TOPIC); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -1040,12 +1042,20 @@ private void handleFilesetCommand() { Command.setAuthenticationMode(auth, userName); + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + + // Handle CommandActions.LIST action separately as it doesn't require the `fileset` if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); newListFilesets(url, ignore, metalake, catalog, schema).handle(); return; } String fileset = name.getFilesetName(); + if (fileset == null) missingEntities.add(CommandEntities.FILESET); + checkEntities(missingEntities); switch (command) { case CommandActions.DETAILS: @@ -1183,4 +1193,11 @@ public String getAuth() { return null; } + + private void checkEntities(List entities) { + if (!entities.isEmpty()) { + System.err.println("Missing required argument(s): " + COMMA_JOINER.join(entities)); + Main.exit(-1); + } + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java index d751d671731..44e5537955f 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestCatalogCommands.java @@ -19,6 +19,7 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; @@ -30,6 +31,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; @@ -37,6 +39,7 @@ import org.apache.gravitino.cli.commands.CatalogDetails; import org.apache.gravitino.cli.commands.CatalogDisable; import org.apache.gravitino.cli.commands.CatalogEnable; +import org.apache.gravitino.cli.commands.Command; import org.apache.gravitino.cli.commands.CreateCatalog; import org.apache.gravitino.cli.commands.DeleteCatalog; import org.apache.gravitino.cli.commands.ListCatalogProperties; @@ -318,6 +321,36 @@ void testUpdateCatalogNameCommand() { verify(mockUpdateName).handle(); } + @Test + @SuppressWarnings("DefaultCharset") + void testCatalogDetailsCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newCatalogDetails( + GravitinoCommandLine.DEFAULT_URL, + false, + Command.OUTPUT_FORMAT_TABLE, + "metalake_demo", + "catalog"); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + "Missing --name option." + + "\n" + + "Missing required argument(s): " + + CommandEntities.CATALOG); + } + @Test void testEnableCatalogCommand() { CatalogEnable mockEnable = mock(CatalogEnable.class); diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java index 2eb4c536480..b6159343ef0 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestColumnCommands.java @@ -28,9 +28,11 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.base.Joiner; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.AddColumn; @@ -64,6 +66,11 @@ void setUp() { System.setErr(new PrintStream(errContent)); } + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + @AfterEach public void restoreStreams() { System.setOut(originalOut); @@ -435,4 +442,208 @@ void testUpdateColumnDefault() { commandLine.handleCommandLine(); verify(mockUpdateDefault).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, + CommandEntities.SCHEMA, + CommandEntities.TABLE, + CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.SCHEMA, CommandEntities.TABLE, CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutTable() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + "catalog", + "schema", + null, + null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.TABLE, CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteColumnCommandWithoutColumn() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema.users"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteColumn( + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + "catalog", + "schema", + "users", + null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.COLUMN))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListColumnCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListColumns( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, CommandEntities.SCHEMA, CommandEntities.TABLE))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListColumnCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListColumns( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA, CommandEntities.TABLE))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListColumnCommandWithoutTable() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(CommandEntities.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.COLUMN, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListColumns( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.TABLE); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java index 314e118c7d7..b46b73cc3dd 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFilesetCommands.java @@ -19,14 +19,22 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.base.Joiner; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateFileset; @@ -38,17 +46,35 @@ import org.apache.gravitino.cli.commands.SetFilesetProperty; import org.apache.gravitino.cli.commands.UpdateFilesetComment; import org.apache.gravitino.cli.commands.UpdateFilesetName; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestFilesetCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -322,4 +348,133 @@ void testRemoveFilesetPropertyCommand() { commandLine.handleCommandLine(); verify(mockSetProperties).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testListFilesetCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListFilesets(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.CATALOG, CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListFilesetCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListFilesets(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testFilesetDetailCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newFilesetDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, CommandEntities.SCHEMA, CommandEntities.FILESET))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testFilesetDetailCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newFilesetDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA, CommandEntities.FILESET))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testFilesetDetailCommandWithoutFileset() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.FILESET, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newFilesetDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.FILESET))); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java index 3f1c4a4cb1e..98e3ea910fb 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java @@ -19,12 +19,18 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.AddRoleToGroup; @@ -34,17 +40,35 @@ import org.apache.gravitino.cli.commands.GroupDetails; import org.apache.gravitino.cli.commands.ListGroups; import org.apache.gravitino.cli.commands.RemoveRoleFromGroup; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestGroupCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -260,4 +284,23 @@ void testAddRolesToGroupCommand() { verify(mockAddSecondRole).handle(); verify(mockAddFirstRole).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testDeleteGroupCommandWithoutGroupOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.GROUP)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.GROUP, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteGroup(GravitinoCommandLine.DEFAULT_URL, false, false, "metalake_demo", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_GROUP); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java index 377e569aa53..1d1ffded0ff 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestMain.java @@ -192,7 +192,6 @@ public void CreateTagWithNoTag() { assertTrue(errContent.toString().contains(ErrorMessages.TAG_EMPTY)); // Expect error } - @Test @SuppressWarnings("DefaultCharset") public void DeleteTagWithNoTag() { String[] args = {"tag", "delete", "--metalake", "metalake_test_no_tag", "-f"}; diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java index 32c289cfd85..c4a8223dd48 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTableCommands.java @@ -19,8 +19,8 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -30,6 +30,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateTable; @@ -451,14 +452,15 @@ void testListTableWithoutCatalog() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) .newListTables(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); - assertTrue( - errContent - .toString() - .contains( - "Missing required argument(s): " - + CommandEntities.CATALOG - + ", " - + CommandEntities.SCHEMA)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.CATALOG + + ", " + + CommandEntities.SCHEMA); } @Test @@ -478,8 +480,13 @@ void testListTableWithoutSchema() { assertThrows(RuntimeException.class, commandLine::handleCommandLine); verify(commandLine, never()) .newListTables(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); - assertTrue( - errContent.toString().contains("Missing required argument(s): " + CommandEntities.SCHEMA)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.SCHEMA); } @Test @@ -498,16 +505,17 @@ void testDetailTableWithoutCatalog() { verify(commandLine, never()) .newTableDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); - assertTrue( - errContent - .toString() - .contains( - "Missing required argument(s): " - + CommandEntities.CATALOG - + ", " - + CommandEntities.SCHEMA - + ", " - + CommandEntities.TABLE)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.CATALOG + + ", " + + CommandEntities.SCHEMA + + ", " + + CommandEntities.TABLE); } @Test @@ -526,14 +534,15 @@ void testDetailTableWithoutSchema() { verify(commandLine, never()) .newTableDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); - assertTrue( - errContent - .toString() - .contains( - "Missing required argument(s): " - + CommandEntities.SCHEMA - + ", " - + CommandEntities.TABLE)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.SCHEMA + + ", " + + CommandEntities.TABLE); } @Test @@ -554,7 +563,12 @@ void testDetailTableWithoutTable() { verify(commandLine, never()) .newTableDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", "schema", null); - assertTrue( - errContent.toString().contains("Missing required argument(s): " + CommandEntities.TABLE)); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + CommandEntities.TABLE); } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java index 58beb02a8d8..8d7ce17bd31 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTagCommands.java @@ -19,15 +19,21 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateTag; @@ -43,6 +49,7 @@ import org.apache.gravitino.cli.commands.UntagEntity; import org.apache.gravitino.cli.commands.UpdateTagComment; import org.apache.gravitino.cli.commands.UpdateTagName; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -51,11 +58,28 @@ class TestTagCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -528,4 +552,22 @@ public boolean matches(String[] argument) { commandLine.handleCommandLine(); verify(mockUntagEntity).handle(); } + + @Test + void testDeleteTagCommandWithoutTagOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.TAG)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TAG, CommandActions.REMOVE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteTag(GravitinoCommandLine.DEFAULT_URL, false, false, "metalake", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_TAG); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java index 50b580eaf72..7fa2e453f32 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestTopicCommands.java @@ -19,12 +19,20 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.base.Joiner; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.CreateTopic; @@ -35,17 +43,35 @@ import org.apache.gravitino.cli.commands.SetTopicProperty; import org.apache.gravitino.cli.commands.TopicDetails; import org.apache.gravitino.cli.commands.UpdateTopicComment; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestTopicCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -271,4 +297,132 @@ void testRemoveTopicPropertyCommand() { commandLine.handleCommandLine(); verify(mockSetProperties).handle(); } + + @Test + @SuppressWarnings("DefaultCharset") + void testListTopicCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListTopics(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.CATALOG, CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testListTopicCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.LIST)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newListTopics(GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testTopicDetailsCommandWithoutCatalog() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTopicDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", null, null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MISSING_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ") + .join( + Arrays.asList( + CommandEntities.CATALOG, CommandEntities.SCHEMA, CommandEntities.TOPIC))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testTopicDetailsCommandWithoutSchema() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTopicDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "catalog", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.SCHEMA, CommandEntities.TOPIC))); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testTopicDetailsCommandWithoutTopic() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("catalog.schema"); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.TOPIC, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newTopicDetails( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "schema", null, null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals( + output, + ErrorMessages.MALFORMED_NAME + + "\n" + + "Missing required argument(s): " + + Joiner.on(", ").join(Arrays.asList(CommandEntities.TOPIC))); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java index e8a1864b9ff..e8630ce9755 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java @@ -19,12 +19,18 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.AddRoleToUser; @@ -34,17 +40,35 @@ import org.apache.gravitino.cli.commands.RemoveRoleFromUser; import org.apache.gravitino.cli.commands.UserAudit; import org.apache.gravitino.cli.commands.UserDetails; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestUserCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + void restoreExitFlg() { + Main.useExit = true; + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -262,4 +286,22 @@ void testAddRolesToUserCommand() { verify(mockAddFirstRole).handle(); verify(mockAddSecondRole).handle(); } + + @Test + void testDeleteUserWithoutUserOption() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.USER)).thenReturn(false); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.USER, CommandActions.DELETE)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newDeleteUser(GravitinoCommandLine.DEFAULT_URL, false, false, "metalake_demo", null); + String output = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(output, ErrorMessages.MISSING_USER); + } } From f7656bec17ed51389206f8044489d543971f419e Mon Sep 17 00:00:00 2001 From: Lord of Abyss <103809695+Abyss-lord@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:09:39 +0800 Subject: [PATCH 4/7] [#5930] improvement(CLI): improve unknown tag output. (#5978) ### What changes were proposed in this pull request? Improve output when CLI add an unknown tags. ### Why are the changes needed? Fix: #5930 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? local test --- .../main/java/org/apache/gravitino/cli/commands/TagEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java index 55bb4b7f436..d2d1cbbe18f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java @@ -98,7 +98,7 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", tagsToAdd); + String all = tagsToAdd.length == 0 ? "nothing" : String.join(",", tagsToAdd); System.out.println(entity + " now tagged with " + all); } From 082bbdc157206b43d07997025acb608aa8478e2a Mon Sep 17 00:00:00 2001 From: FANNG Date: Thu, 26 Dec 2024 12:19:27 +0800 Subject: [PATCH 5/7] [#5620] feat(fileset): Support credential vending for fileset catalog (#5682) ### What changes were proposed in this pull request? Support credential vending for fileset catalog 1. add `credential-providers` properties for the fileset catalog, schema, and fileset. 2. try to get `credential-providers` from the order of fileset, schema, and catalog. 3. The user could set multi-credential providers ### Why are the changes needed? Fix: #5620 ### Does this PR introduce _any_ user-facing change? will add document after this PR is merged ### How was this patch tested? Add IT and test with local setup Gravitino server --- bundles/aws-bundle/build.gradle.kts | 2 + .../credential/CredentialConstants.java | 1 + .../HadoopCatalogPropertiesMetadata.java | 2 + .../HadoopFilesetPropertiesMetadata.java | 2 + .../HadoopSchemaPropertiesMetadata.java | 2 + .../hadoop/SecureHadoopCatalogOperations.java | 63 +++++-- .../test/FilesetCatalogCredentialIT.java | 160 ++++++++++++++++++ .../org/apache/gravitino/GravitinoEnv.java | 15 +- .../gravitino/catalog/CatalogManager.java | 5 + .../gravitino/catalog/CredentialManager.java | 53 ------ .../gravitino/connector/BaseCatalog.java | 19 +++ .../connector/credential/PathContext.java | 63 +++++++ .../SupportsPathBasedCredentials.java | 43 +++++ .../credential/CatalogCredentialManager.java | 70 ++++++++ .../CredentialOperationDispatcher.java | 124 ++++++++++++++ .../credential/CredentialPrivilege.java | 26 +++ .../gravitino/credential/CredentialUtils.java | 63 ++++++- .../credential/config/CredentialConfig.java | 42 +++++ .../credential/Dummy2CredentialProvider.java | 89 ++++++++++ .../credential/TestCredentialUtils.java | 66 ++++++++ ...he.gravitino.credential.CredentialProvider | 3 +- .../gravitino/server/GravitinoServer.java | 6 +- .../MetadataObjectCredentialOperations.java | 24 ++- ...estMetadataObjectCredentialOperations.java | 13 +- 24 files changed, 867 insertions(+), 89 deletions(-) create mode 100644 clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java delete mode 100644 core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java create mode 100644 core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java create mode 100644 core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java create mode 100644 core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java create mode 100644 core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java create mode 100644 core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts index 94c7d1cb2ce..3af5c8b4f38 100644 --- a/bundles/aws-bundle/build.gradle.kts +++ b/bundles/aws-bundle/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { implementation(libs.aws.iam) implementation(libs.aws.policy) implementation(libs.aws.sts) + implementation(libs.commons.lang3) implementation(libs.hadoop3.aws) implementation(project(":catalogs:catalog-common")) { exclude("*") @@ -46,6 +47,7 @@ dependencies { tasks.withType(ShadowJar::class.java) { isZip64 = true configurations = listOf(project.configurations.runtimeClasspath.get()) + relocate("org.apache.commons", "org.apache.gravitino.aws.shaded.org.apache.commons") archiveClassifier.set("") } diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index 29f9241c890..d2753f24b5e 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -21,6 +21,7 @@ public class CredentialConstants { public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; + public static final String CREDENTIAL_PROVIDERS = "credential-providers"; public static final String S3_TOKEN_CREDENTIAL_PROVIDER = "s3-token"; public static final String S3_TOKEN_EXPIRE_IN_SECS = "s3-token-expire-in-secs"; diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 397e13aa4af..22cf0d5b2cd 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -27,6 +27,7 @@ import org.apache.gravitino.catalog.hadoop.fs.LocalFileSystemProvider; import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.config.CredentialConfig; public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata { @@ -84,6 +85,7 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada // The following two are about authentication. .putAll(KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) + .putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES) .build(); @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java index 250a48d292f..84862dd0941 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetPropertiesMetadata.java @@ -24,6 +24,7 @@ import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import org.apache.gravitino.connector.BasePropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.config.CredentialConfig; public class HadoopFilesetPropertiesMetadata extends BasePropertiesMetadata { @@ -32,6 +33,7 @@ protected Map> specificPropertyEntries() { ImmutableMap.Builder> builder = ImmutableMap.builder(); builder.putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES); builder.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES); + builder.putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES); return builder.build(); } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java index 8892433ac6c..9028cc48f3b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopSchemaPropertiesMetadata.java @@ -24,6 +24,7 @@ import org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig; import org.apache.gravitino.connector.BasePropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.config.CredentialConfig; public class HadoopSchemaPropertiesMetadata extends BasePropertiesMetadata { @@ -49,6 +50,7 @@ public class HadoopSchemaPropertiesMetadata extends BasePropertiesMetadata { false /* hidden */)) .putAll(KerberosConfig.KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) + .putAll(CredentialConfig.CREDENTIAL_PROPERTY_ENTRIES) .build(); @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java index 2180e45d423..7ae10805b5b 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java @@ -20,11 +20,16 @@ package org.apache.gravitino.catalog.hadoop; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import javax.security.auth.Subject; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityStore; @@ -38,6 +43,9 @@ import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.SupportsSchemas; +import org.apache.gravitino.connector.credential.PathContext; +import org.apache.gravitino.connector.credential.SupportsPathBasedCredentials; +import org.apache.gravitino.credential.CredentialUtils; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -50,13 +58,14 @@ import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.meta.FilesetEntity; import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.PrincipalUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @SuppressWarnings("removal") public class SecureHadoopCatalogOperations - implements CatalogOperations, SupportsSchemas, FilesetCatalog { + implements CatalogOperations, SupportsSchemas, FilesetCatalog, SupportsPathBasedCredentials { public static final Logger LOG = LoggerFactory.getLogger(SecureHadoopCatalogOperations.class); @@ -66,6 +75,8 @@ public class SecureHadoopCatalogOperations private UserContext catalogUserContext; + private Map catalogProperties; + public SecureHadoopCatalogOperations() { this.hadoopCatalogOperations = new HadoopCatalogOperations(); } @@ -74,6 +85,20 @@ public SecureHadoopCatalogOperations(EntityStore store) { this.hadoopCatalogOperations = new HadoopCatalogOperations(store); } + @Override + public void initialize( + Map config, CatalogInfo info, HasPropertyMetadata propertiesMetadata) + throws RuntimeException { + hadoopCatalogOperations.initialize(config, info, propertiesMetadata); + this.catalogUserContext = + UserContext.getUserContext( + NameIdentifier.of(info.namespace(), info.name()), + config, + hadoopCatalogOperations.getHadoopConf(), + info); + this.catalogProperties = info.properties(); + } + @VisibleForTesting public HadoopCatalogOperations getBaseHadoopCatalogOperations() { return hadoopCatalogOperations; @@ -163,19 +188,6 @@ public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmpty } } - @Override - public void initialize( - Map config, CatalogInfo info, HasPropertyMetadata propertiesMetadata) - throws RuntimeException { - hadoopCatalogOperations.initialize(config, info, propertiesMetadata); - catalogUserContext = - UserContext.getUserContext( - NameIdentifier.of(info.namespace(), info.name()), - config, - hadoopCatalogOperations.getHadoopConf(), - info); - } - @Override public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) throws NoSuchFilesetException, IllegalArgumentException { @@ -245,6 +257,29 @@ public void testConnection( hadoopCatalogOperations.testConnection(catalogIdent, type, provider, comment, properties); } + @Override + public List getPathContext(NameIdentifier filesetIdentifier) { + Fileset fileset = loadFileset(filesetIdentifier); + String path = fileset.storageLocation(); + Preconditions.checkState( + StringUtils.isNotBlank(path), "The location of fileset should not be empty."); + + Set providers = + CredentialUtils.getCredentialProvidersByOrder( + () -> fileset.properties(), + () -> { + Namespace namespace = filesetIdentifier.namespace(); + NameIdentifier schemaIdentifier = + NameIdentifierUtil.ofSchema( + namespace.level(0), namespace.level(1), namespace.level(2)); + return loadSchema(schemaIdentifier).properties(); + }, + () -> catalogProperties); + return providers.stream() + .map(provider -> new PathContext(path, provider)) + .collect(Collectors.toList()); + } + /** * Add the user to the subject so that we can get the last user in the subject. Hadoop catalog * uses this method to pass api user from the client side, so that we can get the user in the diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java new file mode 100644 index 00000000000..94239fef28f --- /dev/null +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.filesystem.hadoop.integration.test; + +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.S3SecretKeyCredential; +import org.apache.gravitino.credential.S3TokenCredential; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.storage.S3Properties; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") +public class FilesetCatalogCredentialIT extends BaseIT { + + private static final Logger LOG = LoggerFactory.getLogger(FilesetCatalogCredentialIT.class); + + public static final String BUCKET_NAME = System.getenv("S3_BUCKET_NAME"); + public static final String S3_ACCESS_KEY = System.getenv("S3_ACCESS_KEY_ID"); + public static final String S3_SECRET_KEY = System.getenv("S3_SECRET_ACCESS_KEY"); + public static final String S3_ROLE_ARN = System.getenv("S3_ROLE_ARN"); + + private String metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); + private String catalogName = GravitinoITUtils.genRandomName("catalog"); + private String schemaName = GravitinoITUtils.genRandomName("schema"); + private GravitinoMetalake metalake; + + @BeforeAll + public void startIntegrationTest() { + // Do nothing + } + + @BeforeAll + public void startUp() throws Exception { + copyBundleJarsToHadoop("aws-bundle"); + // Need to download jars to gravitino server + super.startIntegrationTest(); + + metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); + catalogName = GravitinoITUtils.genRandomName("catalog"); + schemaName = GravitinoITUtils.genRandomName("schema"); + + Assertions.assertFalse(client.metalakeExists(metalakeName)); + metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); + Assertions.assertTrue(client.metalakeExists(metalakeName)); + + Map properties = Maps.newHashMap(); + properties.put(FILESYSTEM_PROVIDERS, "s3"); + properties.put( + CredentialConstants.CREDENTIAL_PROVIDERS, + S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE + + "," + + S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE); + properties.put( + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE); + properties.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, S3_ACCESS_KEY); + properties.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, S3_SECRET_KEY); + properties.put(S3Properties.GRAVITINO_S3_ENDPOINT, "s3.ap-southeast-2.amazonaws.com"); + properties.put(S3Properties.GRAVITINO_S3_REGION, "ap-southeast-2"); + properties.put(S3Properties.GRAVITINO_S3_ROLE_ARN, S3_ROLE_ARN); + + Catalog catalog = + metalake.createCatalog( + catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + + catalog.asSchemas().createSchema(schemaName, "schema comment", properties); + Assertions.assertTrue(catalog.asSchemas().schemaExists(schemaName)); + } + + @AfterAll + public void tearDown() throws IOException { + Catalog catalog = metalake.loadCatalog(catalogName); + catalog.asSchemas().dropSchema(schemaName, true); + metalake.dropCatalog(catalogName, true); + client.dropMetalake(metalakeName, true); + + if (client != null) { + client.close(); + client = null; + } + + try { + closer.close(); + } catch (Exception e) { + LOG.error("Exception in closing CloseableGroup", e); + } + } + + protected String genStorageLocation(String fileset) { + return String.format("s3a://%s/%s", BUCKET_NAME, fileset); + } + + @Test + void testGetCatalogCredential() { + Catalog catalog = metalake.loadCatalog(catalogName); + Credential[] credentials = catalog.supportsCredentials().getCredentials(); + Assertions.assertEquals(1, credentials.length); + Assertions.assertTrue(credentials[0] instanceof S3SecretKeyCredential); + } + + @Test + void testGetFilesetCredential() { + String filesetName = GravitinoITUtils.genRandomName("test_fileset_credential"); + NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); + Catalog catalog = metalake.loadCatalog(catalogName); + String storageLocation = genStorageLocation(filesetName); + catalog + .asFilesetCatalog() + .createFileset( + filesetIdent, + "fileset comment", + Fileset.Type.MANAGED, + storageLocation, + ImmutableMap.of( + CredentialConstants.CREDENTIAL_PROVIDERS, + S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE)); + + Fileset fileset = catalog.asFilesetCatalog().loadFileset(filesetIdent); + Credential[] credentials = fileset.supportsCredentials().getCredentials(); + Assertions.assertEquals(1, credentials.length); + Assertions.assertTrue(credentials[0] instanceof S3TokenCredential); + } +} diff --git a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java index 96c60b834fc..57f04a0cfbf 100644 --- a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java +++ b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java @@ -28,7 +28,6 @@ import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.catalog.CatalogManager; import org.apache.gravitino.catalog.CatalogNormalizeDispatcher; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.catalog.FilesetDispatcher; import org.apache.gravitino.catalog.FilesetNormalizeDispatcher; import org.apache.gravitino.catalog.FilesetOperationDispatcher; @@ -47,6 +46,7 @@ import org.apache.gravitino.catalog.TopicDispatcher; import org.apache.gravitino.catalog.TopicNormalizeDispatcher; import org.apache.gravitino.catalog.TopicOperationDispatcher; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.hook.AccessControlHookDispatcher; import org.apache.gravitino.hook.CatalogHookDispatcher; import org.apache.gravitino.hook.FilesetHookDispatcher; @@ -108,7 +108,7 @@ public class GravitinoEnv { private MetalakeDispatcher metalakeDispatcher; - private CredentialManager credentialManager; + private CredentialOperationDispatcher credentialOperationDispatcher; private TagDispatcher tagDispatcher; @@ -264,12 +264,12 @@ public MetalakeDispatcher metalakeDispatcher() { } /** - * Get the {@link CredentialManager} associated with the Gravitino environment. + * Get the {@link CredentialOperationDispatcher} associated with the Gravitino environment. * - * @return The {@link CredentialManager} instance. + * @return The {@link CredentialOperationDispatcher} instance. */ - public CredentialManager credentialManager() { - return credentialManager; + public CredentialOperationDispatcher credentialOperationDispatcher() { + return credentialOperationDispatcher; } /** @@ -432,7 +432,8 @@ private void initGravitinoServerComponents() { new CatalogNormalizeDispatcher(catalogHookDispatcher); this.catalogDispatcher = new CatalogEventDispatcher(eventBus, catalogNormalizeDispatcher); - this.credentialManager = new CredentialManager(catalogManager, entityStore, idGenerator); + this.credentialOperationDispatcher = + new CredentialOperationDispatcher(catalogManager, entityStore, idGenerator); SchemaOperationDispatcher schemaOperationDispatcher = new SchemaOperationDispatcher(catalogManager, entityStore, idGenerator); diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index 4a46952f87e..1e9c1d9d94f 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -126,6 +126,7 @@ public static void checkCatalogInUse(EntityStore store, NameIdentifier ident) /** Wrapper class for a catalog instance and its class loader. */ public static class CatalogWrapper { + private BaseCatalog catalog; private IsolatedClassLoader classLoader; @@ -169,6 +170,10 @@ public R doWithFilesetOps(ThrowableFunction fn) throws Ex }); } + public R doWithCredentialOps(ThrowableFunction fn) throws Exception { + return classLoader.withClassLoader(cl -> fn.apply(catalog)); + } + public R doWithTopicOps(ThrowableFunction fn) throws Exception { return classLoader.withClassLoader( cl -> { diff --git a/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java b/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java deleted file mode 100644 index 808fc96fb0a..00000000000 --- a/core/src/main/java/org/apache/gravitino/catalog/CredentialManager.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.catalog; - -import java.util.List; -import org.apache.commons.lang3.NotImplementedException; -import org.apache.gravitino.EntityStore; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.connector.BaseCatalog; -import org.apache.gravitino.credential.Credential; -import org.apache.gravitino.exceptions.NoSuchCatalogException; -import org.apache.gravitino.storage.IdGenerator; -import org.apache.gravitino.utils.NameIdentifierUtil; - -/** Get credentials with the specific catalog classloader. */ -public class CredentialManager extends OperationDispatcher { - - public CredentialManager( - CatalogManager catalogManager, EntityStore store, IdGenerator idGenerator) { - super(catalogManager, store, idGenerator); - } - - public List getCredentials(NameIdentifier identifier) { - return doWithCatalog( - NameIdentifierUtil.getCatalogIdentifier(identifier), - c -> getCredentials(c.catalog(), identifier), - NoSuchCatalogException.class); - } - - private List getCredentials(BaseCatalog catalog, NameIdentifier identifier) { - throw new NotImplementedException( - String.format( - "Load credentials is not implemented for catalog: %s, identifier: %s", - catalog.name(), identifier)); - } -} diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index 218c2a428b3..14b1912b4d6 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -31,6 +31,7 @@ import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.connector.authorization.BaseAuthorization; import org.apache.gravitino.connector.capability.Capability; +import org.apache.gravitino.credential.CatalogCredentialManager; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.utils.IsolatedClassLoader; import org.slf4j.Logger; @@ -51,6 +52,7 @@ @Evolving public abstract class BaseCatalog implements Catalog, CatalogProvider, HasPropertyMetadata, Closeable { + private static final Logger LOG = LoggerFactory.getLogger(BaseCatalog.class); // This variable is used as a key in properties of catalogs to inject custom operation to @@ -72,6 +74,8 @@ public abstract class BaseCatalog private volatile Map properties; + private volatile CatalogCredentialManager catalogCredentialManager; + private static String ENTITY_IS_NOT_SET = "entity is not set"; // Any Gravitino configuration that starts with this prefix will be trim and passed to the @@ -225,6 +229,10 @@ public void close() throws IOException { authorizationPlugin.close(); authorizationPlugin = null; } + if (catalogCredentialManager != null) { + catalogCredentialManager.close(); + catalogCredentialManager = null; + } } public Capability capability() { @@ -239,6 +247,17 @@ public Capability capability() { return capability; } + public CatalogCredentialManager catalogCredentialManager() { + if (catalogCredentialManager == null) { + synchronized (this) { + if (catalogCredentialManager == null) { + this.catalogCredentialManager = new CatalogCredentialManager(name(), properties()); + } + } + } + return catalogCredentialManager; + } + private CatalogOperations createOps(Map conf) { String customCatalogOperationClass = conf.get(CATALOG_OPERATION_IMPL); return Optional.ofNullable(customCatalogOperationClass) diff --git a/core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java b/core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java new file mode 100644 index 00000000000..5c520d6bfda --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/connector/credential/PathContext.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.connector.credential; + +import org.apache.gravitino.annotation.DeveloperApi; + +/** + * The {@code PathContext} class represents the path and its associated credential type to generate + * a credential for {@link org.apache.gravitino.credential.CredentialOperationDispatcher}. + */ +@DeveloperApi +public class PathContext { + + private final String path; + + private final String credentialType; + + /** + * Constructs a new {@code PathContext} instance with the given path and credential type. + * + * @param path The path string. + * @param credentialType The type of the credential. + */ + public PathContext(String path, String credentialType) { + this.path = path; + this.credentialType = credentialType; + } + + /** + * Gets the path string. + * + * @return The path associated with this instance. + */ + public String path() { + return path; + } + + /** + * Gets the credential type. + * + * @return The credential type associated with this instance. + */ + public String credentialType() { + return credentialType; + } +} diff --git a/core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java b/core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java new file mode 100644 index 00000000000..93e08a39069 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/connector/credential/SupportsPathBasedCredentials.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.connector.credential; + +import java.util.List; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.annotation.DeveloperApi; + +/** The catalog operation should implement this interface to generate the path based credentials. */ +@DeveloperApi +public interface SupportsPathBasedCredentials { + + /** + * Get {@link PathContext} lists. + * + *

In most cases there will be only one element in the list. For catalogs which support multi + * locations like fileset, there may be multiple elements. + * + *

The name identifier is the identifier of the resource like fileset, table, etc. not include + * metalake, catalog, schema. + * + * @param nameIdentifier, The identifier for fileset, table, etc. + * @return A list of {@link PathContext} + */ + List getPathContext(NameIdentifier nameIdentifier); +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java new file mode 100644 index 00000000000..2fe6fedccd9 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manage lifetime of the credential provider in one catalog, dispatch credential request to the + * corresponding credential provider. + */ +public class CatalogCredentialManager implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(CatalogCredentialManager.class); + + private final String catalogName; + private final Map credentialProviders; + + public CatalogCredentialManager(String catalogName, Map catalogProperties) { + this.catalogName = catalogName; + this.credentialProviders = CredentialUtils.loadCredentialProviders(catalogProperties); + } + + public Credential getCredential(String credentialType, CredentialContext context) { + // todo: add credential cache + Preconditions.checkState( + credentialProviders.containsKey(credentialType), + String.format("Credential %s not found", credentialType)); + return credentialProviders.get(credentialType).getCredential(context); + } + + @Override + public void close() { + credentialProviders + .values() + .forEach( + credentialProvider -> { + try { + credentialProvider.close(); + } catch (IOException e) { + LOG.warn( + "Close credential provider failed, catalog: {}, credential provider: {}", + catalogName, + credentialProvider.credentialType(), + e); + } + }); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java new file mode 100644 index 00000000000..2ec76aeb4ad --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialOperationDispatcher.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import javax.ws.rs.NotAuthorizedException; +import javax.ws.rs.NotSupportedException; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.catalog.CatalogManager; +import org.apache.gravitino.catalog.OperationDispatcher; +import org.apache.gravitino.connector.BaseCatalog; +import org.apache.gravitino.connector.credential.PathContext; +import org.apache.gravitino.connector.credential.SupportsPathBasedCredentials; +import org.apache.gravitino.exceptions.NoSuchCatalogException; +import org.apache.gravitino.storage.IdGenerator; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.PrincipalUtils; + +/** Get credentials with the specific catalog classloader. */ +public class CredentialOperationDispatcher extends OperationDispatcher { + + public CredentialOperationDispatcher( + CatalogManager catalogManager, EntityStore store, IdGenerator idGenerator) { + super(catalogManager, store, idGenerator); + } + + public List getCredentials(NameIdentifier identifier) { + CredentialPrivilege privilege = + getCredentialPrivilege(PrincipalUtils.getCurrentUserName(), identifier); + return doWithCatalog( + NameIdentifierUtil.getCatalogIdentifier(identifier), + catalogWrapper -> + catalogWrapper.doWithCredentialOps( + baseCatalog -> getCredentials(baseCatalog, identifier, privilege)), + NoSuchCatalogException.class); + } + + private List getCredentials( + BaseCatalog baseCatalog, NameIdentifier nameIdentifier, CredentialPrivilege privilege) { + Map contexts = + getCredentialContexts(baseCatalog, nameIdentifier, privilege); + return contexts.entrySet().stream() + .map( + entry -> + baseCatalog + .catalogCredentialManager() + .getCredential(entry.getKey(), entry.getValue())) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private Map getCredentialContexts( + BaseCatalog baseCatalog, NameIdentifier nameIdentifier, CredentialPrivilege privilege) { + if (nameIdentifier.equals(NameIdentifierUtil.getCatalogIdentifier(nameIdentifier))) { + return getCatalogCredentialContexts(baseCatalog.properties()); + } + + if (baseCatalog.ops() instanceof SupportsPathBasedCredentials) { + List pathContexts = + ((SupportsPathBasedCredentials) baseCatalog.ops()).getPathContext(nameIdentifier); + return getPathBasedCredentialContexts(privilege, pathContexts); + } + throw new NotSupportedException( + String.format("Catalog %s doesn't support generate credentials", baseCatalog.name())); + } + + private Map getCatalogCredentialContexts( + Map catalogProperties) { + CatalogCredentialContext context = + new CatalogCredentialContext(PrincipalUtils.getCurrentUserName()); + Set providers = CredentialUtils.getCredentialProvidersByOrder(() -> catalogProperties); + return providers.stream().collect(Collectors.toMap(provider -> provider, provider -> context)); + } + + public static Map getPathBasedCredentialContexts( + CredentialPrivilege privilege, List pathContexts) { + return pathContexts.stream() + .collect( + Collectors.toMap( + pathContext -> pathContext.credentialType(), + pathContext -> { + String path = pathContext.path(); + Set writePaths = new HashSet<>(); + Set readPaths = new HashSet<>(); + if (CredentialPrivilege.WRITE.equals(privilege)) { + writePaths.add(path); + } else { + readPaths.add(path); + } + return new PathBasedCredentialContext( + PrincipalUtils.getCurrentUserName(), writePaths, readPaths); + })); + } + + @SuppressWarnings("UnusedVariable") + private CredentialPrivilege getCredentialPrivilege(String user, NameIdentifier identifier) + throws NotAuthorizedException { + // TODO: will implement in another PR + return CredentialPrivilege.WRITE; + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java b/core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java new file mode 100644 index 00000000000..3ff77cd3e8f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialPrivilege.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +/** Represents the privilege to get credential from credential providers. */ +public enum CredentialPrivilege { + READ, + WRITE, +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java index 09439d58ae8..9a202ec9747 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java @@ -19,14 +19,75 @@ package org.apache.gravitino.credential; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.gravitino.utils.PrincipalUtils; public class CredentialUtils { + + private static final Splitter splitter = Splitter.on(","); + public static Credential vendCredential(CredentialProvider credentialProvider, String[] path) { PathBasedCredentialContext pathBasedCredentialContext = new PathBasedCredentialContext( - PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path), ImmutableSet.of()); + PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path), Collections.emptySet()); return credentialProvider.getCredential(pathBasedCredentialContext); } + + public static Map loadCredentialProviders( + Map catalogProperties) { + Set credentialProviders = + CredentialUtils.getCredentialProvidersByOrder(() -> catalogProperties); + + return credentialProviders.stream() + .collect( + Collectors.toMap( + String::toString, + credentialType -> + CredentialProviderFactory.create(credentialType, catalogProperties))); + } + + /** + * Get Credential providers from properties supplier. + * + *

If there are multiple properties suppliers, will try to get the credential providers in the + * input order. + * + * @param propertiesSuppliers The properties suppliers. + * @return A set of credential providers. + */ + public static Set getCredentialProvidersByOrder( + Supplier>... propertiesSuppliers) { + + for (Supplier> supplier : propertiesSuppliers) { + Map properties = supplier.get(); + Set providers = getCredentialProvidersFromProperties(properties); + if (!providers.isEmpty()) { + return providers; + } + } + + return Collections.emptySet(); + } + + private static Set getCredentialProvidersFromProperties(Map properties) { + if (properties == null) { + return Collections.emptySet(); + } + + String providers = properties.get(CredentialConstants.CREDENTIAL_PROVIDERS); + if (providers == null) { + return Collections.emptySet(); + } + return splitter + .trimResults() + .omitEmptyStrings() + .splitToStream(providers) + .collect(Collectors.toSet()); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java new file mode 100644 index 00000000000..d8823417cda --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential.config; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.connector.PropertyEntry; +import org.apache.gravitino.credential.CredentialConstants; + +public class CredentialConfig { + + public static final Map> CREDENTIAL_PROPERTY_ENTRIES = + new ImmutableMap.Builder>() + .put( + CredentialConstants.CREDENTIAL_PROVIDERS, + PropertyEntry.booleanPropertyEntry( + CredentialConstants.CREDENTIAL_PROVIDERS, + "Credential providers for the Gravitino catalog, schema, fileset, table, etc.", + false /* required */, + false /* immutable */, + null /* default value */, + false /* hidden */, + false /* reserved */)) + .build(); +} diff --git a/core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java b/core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java new file mode 100644 index 00000000000..63f63d61d0b --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/credential/Dummy2CredentialProvider.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.Set; +import javax.ws.rs.NotSupportedException; +import lombok.Getter; + +public class Dummy2CredentialProvider implements CredentialProvider { + Map properties; + static final String CREDENTIAL_TYPE = "dummy2"; + + @Override + public void initialize(Map properties) { + this.properties = properties; + } + + @Override + public void close() {} + + @Override + public String credentialType() { + return CREDENTIAL_TYPE; + } + + @Override + public Credential getCredential(CredentialContext context) { + Preconditions.checkArgument( + context instanceof PathBasedCredentialContext + || context instanceof CatalogCredentialContext, + "Doesn't support context: " + context.getClass().getSimpleName()); + if (context instanceof PathBasedCredentialContext) { + return new Dummy2Credential((PathBasedCredentialContext) context); + } + return null; + } + + public static class Dummy2Credential implements Credential { + + @Getter private Set writeLocations; + @Getter private Set readLocations; + + public Dummy2Credential(PathBasedCredentialContext locationContext) { + this.writeLocations = locationContext.getWritePaths(); + this.readLocations = locationContext.getReadPaths(); + } + + @Override + public String credentialType() { + return Dummy2CredentialProvider.CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return 0; + } + + @Override + public Map credentialInfo() { + return ImmutableMap.of( + "writeLocation", writeLocations.toString(), "readLocation", readLocations.toString()); + } + + @Override + public void initialize(Map credentialInfo, long expireTimeInMs) { + throw new NotSupportedException(); + } + } +} diff --git a/core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java b/core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java new file mode 100644 index 00000000000..c31affdc157 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/credential/TestCredentialUtils.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testcontainers.shaded.com.google.common.collect.ImmutableSet; + +public class TestCredentialUtils { + + @Test + void testLoadCredentialProviders() { + Map catalogProperties = + ImmutableMap.of( + CredentialConstants.CREDENTIAL_PROVIDERS, + DummyCredentialProvider.CREDENTIAL_TYPE + + "," + + Dummy2CredentialProvider.CREDENTIAL_TYPE); + Map providers = + CredentialUtils.loadCredentialProviders(catalogProperties); + Assertions.assertTrue(providers.size() == 2); + + Assertions.assertTrue(providers.containsKey(DummyCredentialProvider.CREDENTIAL_TYPE)); + Assertions.assertTrue( + DummyCredentialProvider.CREDENTIAL_TYPE.equals( + providers.get(DummyCredentialProvider.CREDENTIAL_TYPE).credentialType())); + Assertions.assertTrue(providers.containsKey(Dummy2CredentialProvider.CREDENTIAL_TYPE)); + Assertions.assertTrue( + Dummy2CredentialProvider.CREDENTIAL_TYPE.equals( + providers.get(Dummy2CredentialProvider.CREDENTIAL_TYPE).credentialType())); + } + + @Test + void testGetCredentialProviders() { + Map filesetProperties = ImmutableMap.of(); + Map schemaProperties = + ImmutableMap.of(CredentialConstants.CREDENTIAL_PROVIDERS, "a,b"); + Map catalogProperties = + ImmutableMap.of(CredentialConstants.CREDENTIAL_PROVIDERS, "a,b,c"); + + Set credentialProviders = + CredentialUtils.getCredentialProvidersByOrder( + () -> filesetProperties, () -> schemaProperties, () -> catalogProperties); + Assertions.assertEquals(credentialProviders, ImmutableSet.of("a", "b")); + } +} diff --git a/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider index cbdbff0bee9..6e1fdde4bdb 100644 --- a/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider +++ b/core/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -16,4 +16,5 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.credential.DummyCredentialProvider \ No newline at end of file +org.apache.gravitino.credential.DummyCredentialProvider +org.apache.gravitino.credential.Dummy2CredentialProvider diff --git a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java index 16a2096f328..63e53aefd59 100644 --- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java @@ -26,12 +26,12 @@ import org.apache.gravitino.Configs; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.catalog.CatalogDispatcher; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.catalog.FilesetDispatcher; import org.apache.gravitino.catalog.PartitionDispatcher; import org.apache.gravitino.catalog.SchemaDispatcher; import org.apache.gravitino.catalog.TableDispatcher; import org.apache.gravitino.catalog.TopicDispatcher; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.metalake.MetalakeDispatcher; import org.apache.gravitino.metrics.MetricsSystem; import org.apache.gravitino.metrics.source.MetricsSource; @@ -115,7 +115,9 @@ protected void configure() { bind(gravitinoEnv.filesetDispatcher()).to(FilesetDispatcher.class).ranked(1); bind(gravitinoEnv.topicDispatcher()).to(TopicDispatcher.class).ranked(1); bind(gravitinoEnv.tagDispatcher()).to(TagDispatcher.class).ranked(1); - bind(gravitinoEnv.credentialManager()).to(CredentialManager.class).ranked(1); + bind(gravitinoEnv.credentialOperationDispatcher()) + .to(CredentialOperationDispatcher.class) + .ranked(1); } }); register(JsonProcessingExceptionMapper.class); diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java index 7c6ea4a8eb7..1046bbba1a5 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/MetadataObjectCredentialOperations.java @@ -21,11 +21,14 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Locale; +import java.util.Set; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; +import javax.ws.rs.NotSupportedException; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -34,8 +37,8 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.dto.credential.CredentialDTO; import org.apache.gravitino.dto.responses.CredentialResponse; import org.apache.gravitino.dto.util.DTOConverters; @@ -51,15 +54,18 @@ public class MetadataObjectCredentialOperations { private static final Logger LOG = LoggerFactory.getLogger(MetadataObjectCredentialOperations.class); - private CredentialManager credentialManager; + private static final Set supportsCredentialMetadataTypes = + ImmutableSet.of(MetadataObject.Type.CATALOG, MetadataObject.Type.FILESET); + + private CredentialOperationDispatcher credentialOperationDispatcher; @SuppressWarnings("unused") @Context private HttpServletRequest httpRequest; @Inject - public MetadataObjectCredentialOperations(CredentialManager dispatcher) { - this.credentialManager = dispatcher; + public MetadataObjectCredentialOperations(CredentialOperationDispatcher dispatcher) { + this.credentialOperationDispatcher = dispatcher; } @GET @@ -83,9 +89,13 @@ public Response getCredentials( MetadataObject object = MetadataObjects.parse( fullName, MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT))); + if (!supportsCredentialOperations(object)) { + throw new NotSupportedException( + "Doesn't support credential operations for metadata object type"); + } NameIdentifier identifier = MetadataObjectUtil.toEntityIdent(metalake, object); - List credentials = credentialManager.getCredentials(identifier); + List credentials = credentialOperationDispatcher.getCredentials(identifier); if (credentials == null) { return Utils.ok(new CredentialResponse(new CredentialDTO[0])); } @@ -97,4 +107,8 @@ public Response getCredentials( return ExceptionHandlers.handleCredentialException(OperationType.GET, fullName, e); } } + + private static boolean supportsCredentialOperations(MetadataObject metadataObject) { + return supportsCredentialMetadataTypes.contains(metadataObject.type()); + } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java index 1ac5d38135d..464ccd86984 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetadataObjectCredentialOperations.java @@ -31,8 +31,8 @@ import javax.ws.rs.core.Response; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; -import org.apache.gravitino.catalog.CredentialManager; import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialOperationDispatcher; import org.apache.gravitino.credential.S3SecretKeyCredential; import org.apache.gravitino.dto.responses.CredentialResponse; import org.apache.gravitino.dto.responses.ErrorConstants; @@ -59,7 +59,8 @@ public HttpServletRequest get() { } } - private CredentialManager credentialManager = mock(CredentialManager.class); + private CredentialOperationDispatcher credentialOperationDispatcher = + mock(CredentialOperationDispatcher.class); private String metalake = "test_metalake"; @@ -78,7 +79,7 @@ protected Application configure() { new AbstractBinder() { @Override protected void configure() { - bind(credentialManager).to(CredentialManager.class).ranked(2); + bind(credentialOperationDispatcher).to(CredentialOperationDispatcher.class).ranked(2); bindFactory(MockServletRequestFactory.class).to(HttpServletRequest.class); } }); @@ -101,7 +102,7 @@ private void testGetCredentialsForObject(MetadataObject metadataObject) { S3SecretKeyCredential credential = new S3SecretKeyCredential("access-id", "secret-key"); // Test return one credential - when(credentialManager.getCredentials(any())).thenReturn(Arrays.asList(credential)); + when(credentialOperationDispatcher.getCredentials(any())).thenReturn(Arrays.asList(credential)); Response response = target(basePath(metalake)) .path(metadataObject.type().toString()) @@ -123,7 +124,7 @@ private void testGetCredentialsForObject(MetadataObject metadataObject) { Assertions.assertEquals(0, credentialToTest.expireTimeInMs()); // Test doesn't return credential - when(credentialManager.getCredentials(any())).thenReturn(null); + when(credentialOperationDispatcher.getCredentials(any())).thenReturn(null); response = target(basePath(metalake)) .path(metadataObject.type().toString()) @@ -140,7 +141,7 @@ private void testGetCredentialsForObject(MetadataObject metadataObject) { // Test throws NoSuchCredentialException doThrow(new NoSuchCredentialException("mock error")) - .when(credentialManager) + .when(credentialOperationDispatcher) .getCredentials(any()); response = target(basePath(metalake)) From a68e5e22addb28448b91ddeb717d52bf7d9f74e0 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Thu, 26 Dec 2024 15:30:55 +0800 Subject: [PATCH 6/7] [#5817] core(feat): Add server-side REST APIs for model management (#5948) ### What changes were proposed in this pull request? This PR adds the server-side REST endpoint for model management. ### Why are the changes needed? This is a part of model management for Gravitino. Fix: #5817 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add UTs for this PR. --- .../apache/gravitino/model/ModelCatalog.java | 8 +- .../catalog/model/ModelCatalogImpl.java | 3 +- .../org.apache.gravitino.CatalogProvider | 2 +- .../apache/gravitino/dto/model/ModelDTO.java | 163 ++++ .../gravitino/dto/model/ModelVersionDTO.java | 184 ++++ .../dto/requests/CatalogCreateRequest.java | 31 +- .../dto/requests/ModelRegisterRequest.java | 54 ++ .../dto/requests/ModelVersionLinkRequest.java | 67 ++ .../dto/responses/ModelResponse.java | 59 ++ .../responses/ModelVersionListResponse.java | 57 ++ .../dto/responses/ModelVersionResponse.java | 59 ++ .../gravitino/dto/util/DTOConverters.java | 37 + .../gravitino/dto/model/TestModelDTO.java | 81 ++ .../dto/model/TestModelVersionDTO.java | 133 +++ .../requests/TestCatalogCreateRequest.java | 15 +- .../requests/TestModelRegisterRequest.java | 47 + .../requests/TestModelVersionLinkRequest.java | 75 ++ .../dto/responses/TestResponses.java | 76 ++ docs/open-api/models.yaml | 561 ++++++++++++ docs/open-api/openapi.yaml | 23 + .../gravitino/server/GravitinoServer.java | 2 + .../server/web/rest/ExceptionHandlers.java | 45 + .../server/web/rest/ModelOperations.java | 411 +++++++++ .../server/web/rest/OperationType.java | 3 + .../server/web/rest/TestModelOperations.java | 843 ++++++++++++++++++ 25 files changed, 3006 insertions(+), 33 deletions(-) create mode 100644 common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java create mode 100644 docs/open-api/models.yaml create mode 100644 server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java create mode 100644 server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java diff --git a/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java b/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java index cea2e94e3c7..3fb39c18aea 100644 --- a/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java +++ b/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java @@ -93,7 +93,10 @@ Model registerModel(NameIdentifier ident, String comment, Map pr * * @param ident The name identifier of the model. * @param uri The model artifact URI. - * @param aliases The aliases of the model version. The alias are optional and can be empty. + * @param aliases The aliases of the model version. The aliases should be unique in this model, + * otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases + * are optional and can be empty. Also, be aware that the alias cannot be a number or a number + * string. * @param comment The comment of the model. The comment is optional and can be null. * @param properties The properties of the model. The properties are optional and can be null or * empty. @@ -198,7 +201,8 @@ default boolean modelVersionExists(NameIdentifier ident, String alias) { * @param uri The URI of the model version artifact. * @param aliases The aliases of the model version. The aliases should be unique in this model, * otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases - * are optional and can be empty. + * are optional and can be empty. Also, be aware that the alias cannot be a number or a number + * string. * @param comment The comment of the model version. The comment is optional and can be null. * @param properties The properties of the model version. The properties are optional and can be * null or empty. diff --git a/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java index 5b90eab7265..545f6482a3f 100644 --- a/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java +++ b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java @@ -19,7 +19,6 @@ package org.apache.gravitino.catalog.model; import java.util.Map; -import org.apache.gravitino.CatalogProvider; import org.apache.gravitino.EntityStore; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.connector.BaseCatalog; @@ -40,7 +39,7 @@ public class ModelCatalogImpl extends BaseCatalog { @Override public String shortName() { - return CatalogProvider.shortNameForManagedCatalog(super.type()); + return "model"; } @Override diff --git a/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider index 37c682aa745..e43f995ea7d 100644 --- a/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider +++ b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider @@ -16,4 +16,4 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.catalog.model.ModelCatalog +org.apache.gravitino.catalog.model.ModelCatalogImpl diff --git a/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java b/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java new file mode 100644 index 00000000000..44688399335 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.model.Model; + +/** Represents a model DTO (Data Transfer Object). */ +@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class ModelDTO implements Model { + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @JsonProperty("latestVersion") + private int latestVersion; + + @JsonProperty("audit") + private AuditDTO audit; + + @Override + public String name() { + return name; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Map properties() { + return properties; + } + + @Override + public int latestVersion() { + return latestVersion; + } + + @Override + public AuditDTO auditInfo() { + return audit; + } + + /** + * Creates a new builder for constructing a Model DTO. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for constructing a Model DTO. */ + public static class Builder { + private String name; + private String comment; + private Map properties; + private int latestVersion; + private AuditDTO audit; + + /** + * Sets the name of the model. + * + * @param name The name of the model. + * @return The builder. + */ + public Builder withName(String name) { + this.name = name; + return this; + } + + /** + * Sets the comment associated with the model. + * + * @param comment The comment associated with the model. + * @return The builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Sets the properties associated with the model. + * + * @param properties The properties associated with the model. + * @return The builder. + */ + public Builder withProperties(Map properties) { + this.properties = properties; + return this; + } + + /** + * Sets the latest version of the model. + * + * @param latestVersion The latest version of the model. + * @return The builder. + */ + public Builder withLatestVersion(int latestVersion) { + this.latestVersion = latestVersion; + return this; + } + + /** + * Sets the audit information associated with the model. + * + * @param audit The audit information associated with the model. + * @return The builder. + */ + public Builder withAudit(AuditDTO audit) { + this.audit = audit; + return this; + } + + /** + * Builds the model DTO. + * + * @return The model DTO. + */ + public ModelDTO build() { + Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be null or empty"); + Preconditions.checkArgument(latestVersion >= 0, "latestVersion cannot be negative"); + Preconditions.checkArgument(audit != null, "audit cannot be null"); + + return new ModelDTO(name, comment, properties, latestVersion, audit); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java b/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java new file mode 100644 index 00000000000..e887ba5bdb2 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Audit; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.model.ModelVersion; + +/** Represents a model version DTO (Data Transfer Object). */ +@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class ModelVersionDTO implements ModelVersion { + + @JsonProperty("version") + private int version; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("aliases") + private String[] aliases; + + @JsonProperty("uri") + private String uri; + + @JsonProperty("properties") + private Map properties; + + @JsonProperty("audit") + private AuditDTO audit; + + @Override + public Audit auditInfo() { + return audit; + } + + @Override + public int version() { + return version; + } + + @Override + public String comment() { + return comment; + } + + @Override + public String[] aliases() { + return aliases; + } + + @Override + public String uri() { + return uri; + } + + @Override + public Map properties() { + return properties; + } + + /** + * Creates a new builder for constructing a Model Version DTO. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for constructing a Model Version DTO. */ + public static class Builder { + private int version; + private String comment; + private String[] aliases; + private String uri; + private Map properties; + private AuditDTO audit; + + /** + * Sets the version number of the model version. + * + * @param version The version number. + * @return The builder. + */ + public Builder withVersion(int version) { + this.version = version; + return this; + } + + /** + * Sets the comment of the model version. + * + * @param comment The comment. + * @return The builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Sets the aliases of the model version. + * + * @param aliases The aliases. + * @return The builder. + */ + public Builder withAliases(String[] aliases) { + this.aliases = aliases; + return this; + } + + /** + * Sets the URI of the model version. + * + * @param uri The URI. + * @return The builder. + */ + public Builder withUri(String uri) { + this.uri = uri; + return this; + } + + /** + * Sets the properties of the model version. + * + * @param properties The properties. + * @return The builder. + */ + public Builder withProperties(Map properties) { + this.properties = properties; + return this; + } + + /** + * Sets the audit information of the model version. + * + * @param audit The audit information. + * @return The builder. + */ + public Builder withAudit(AuditDTO audit) { + this.audit = audit; + return this; + } + + /** + * Builds the Model Version DTO. + * + * @return The Model Version DTO. + */ + public ModelVersionDTO build() { + Preconditions.checkArgument(version >= 0, "Version must be non-negative"); + Preconditions.checkArgument(StringUtils.isNotBlank(uri), "URI cannot be null or empty"); + Preconditions.checkArgument(audit != null, "Audit cannot be null"); + + return new ModelVersionDTO(version, comment, aliases, uri, properties, audit); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java index 3da6579676d..d543ddb1649 100644 --- a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java +++ b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.dto.requests; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; import com.google.common.base.Preconditions; import java.util.Map; import javax.annotation.Nullable; @@ -54,11 +54,6 @@ public class CatalogCreateRequest implements RESTRequest { @JsonProperty("properties") private final Map properties; - /** Default constructor for CatalogCreateRequest. */ - public CatalogCreateRequest() { - this(null, null, null, null, null); - } - /** * Constructor for CatalogCreateRequest. * @@ -68,34 +63,24 @@ public CatalogCreateRequest() { * @param comment The comment for the catalog. * @param properties The properties for the catalog. */ + @JsonCreator public CatalogCreateRequest( - String name, - Catalog.Type type, - String provider, - String comment, - Map properties) { + @JsonProperty("name") String name, + @JsonProperty("type") Catalog.Type type, + @JsonProperty("provider") String provider, + @JsonProperty("comment") String comment, + @JsonProperty("properties") Map properties) { this.name = name; this.type = type; - this.provider = provider; this.comment = comment; this.properties = properties; - } - /** - * Sets the provider of the catalog if it is null. The value of provider in the request can be - * null if the catalog is a managed catalog. For such request, the value will be set when it is - * deserialized. - * - * @param provider The provider of the catalog. - */ - @JsonSetter(value = "provider") - public void setProvider(String provider) { if (StringUtils.isNotBlank(provider)) { this.provider = provider; } else if (type != null && type.supportsManagedCatalog()) { this.provider = CatalogProvider.shortNameForManagedCatalog(type); } else { - throw new IllegalStateException( + throw new IllegalArgumentException( "Provider cannot be null for catalog type " + type + " that doesn't support managed catalog"); diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java new file mode 100644 index 00000000000..b9cd1916174 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to register a model. */ +@Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class ModelRegisterRequest implements RESTRequest { + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(name), "\"name\" field is required and cannot be empty"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java new file mode 100644 index 00000000000..24e5932932c --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to link a model version. */ +@Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class ModelVersionLinkRequest implements RESTRequest { + + @JsonProperty("uri") + private String uri; + + @JsonProperty("aliases") + private String[] aliases; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(uri), "\"uri\" field is required and cannot be empty"); + + if (aliases != null && aliases.length > 0) { + for (String alias : aliases) { + Preconditions.checkArgument( + StringUtils.isNotBlank(alias), "alias must not be null or empty"); + Preconditions.checkArgument( + !NumberUtils.isCreatable(alias), "alias must not be a number or a number string"); + } + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java new file mode 100644 index 00000000000..ac51cdd647c --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.model.ModelDTO; + +/** Response for model response. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelResponse extends BaseResponse { + + @JsonProperty("model") + private final ModelDTO model; + + /** + * Constructor for ModelResponse. + * + * @param model The model DTO object. + */ + public ModelResponse(ModelDTO model) { + super(0); + this.model = model; + } + + /** Default constructor for ModelResponse. (Used for Jackson deserialization.) */ + public ModelResponse() { + super(); + this.model = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(model != null, "model must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java new file mode 100644 index 00000000000..4d3551e1397 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** Represents a response for a list of model versions. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelVersionListResponse extends BaseResponse { + + @JsonProperty("versions") + private int[] versions; + + /** + * Constructor for ModelVersionListResponse. + * + * @param versions The list of model versions. + */ + public ModelVersionListResponse(int[] versions) { + super(0); + this.versions = versions; + } + + /** Default constructor for ModelVersionListResponse. (Used for Jackson deserialization.) */ + public ModelVersionListResponse() { + super(); + this.versions = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + Preconditions.checkArgument(versions != null, "versions cannot be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java new file mode 100644 index 00000000000..8b21472833b --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.model.ModelVersionDTO; + +/** Represents a response for a model version. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelVersionResponse extends BaseResponse { + + @JsonProperty("modelVersion") + private final ModelVersionDTO modelVersion; + + /** + * Constructor for ModelVersionResponse. + * + * @param modelVersion The model version DTO object. + */ + public ModelVersionResponse(ModelVersionDTO modelVersion) { + super(0); + this.modelVersion = modelVersion; + } + + /** Default constructor for ModelVersionResponse. (Used for Jackson deserialization.) */ + public ModelVersionResponse() { + super(); + this.modelVersion = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(modelVersion != null, "modelVersion must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java index 254de8c3245..ce63398e605 100644 --- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java @@ -51,6 +51,8 @@ import org.apache.gravitino.dto.credential.CredentialDTO; import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.messaging.TopicDTO; +import org.apache.gravitino.dto.model.ModelDTO; +import org.apache.gravitino.dto.model.ModelVersionDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.DistributionDTO; import org.apache.gravitino.dto.rel.SortOrderDTO; @@ -80,6 +82,8 @@ import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.messaging.Topic; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.expressions.Expression; @@ -629,6 +633,39 @@ public static TopicDTO toDTO(Topic topic) { .build(); } + /** + * Converts a Model to a ModelDTO. + * + * @param model The model to be converted. + * @return The model DTO. + */ + public static ModelDTO toDTO(Model model) { + return ModelDTO.builder() + .withName(model.name()) + .withComment(model.comment()) + .withProperties(model.properties()) + .withLatestVersion(model.latestVersion()) + .withAudit(toDTO(model.auditInfo())) + .build(); + } + + /** + * Converts a ModelVersion to a ModelVersionDTO. + * + * @param modelVersion The model version to be converted. + * @return The model version DTO. + */ + public static ModelVersionDTO toDTO(ModelVersion modelVersion) { + return ModelVersionDTO.builder() + .withVersion(modelVersion.version()) + .withComment(modelVersion.comment()) + .withAliases(modelVersion.aliases()) + .withUri(modelVersion.uri()) + .withProperties(modelVersion.properties()) + .withAudit(toDTO(modelVersion.auditInfo())) + .build(); + } + /** * Converts an array of Columns to an array of ColumnDTOs. * diff --git a/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java b/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java new file mode 100644 index 00000000000..39e4628eca2 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelDTO { + + @Test + public void testModelSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + Map props = ImmutableMap.of("key", "value"); + + ModelDTO modelDTO = + ModelDTO.builder() + .withName("model_test") + .withComment("model comment") + .withLatestVersion(0) + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(modelDTO); + ModelDTO deserModelDTO = JsonUtils.objectMapper().readValue(serJson, ModelDTO.class); + Assertions.assertEquals(modelDTO, deserModelDTO); + + // Test with null comment and properties + ModelDTO modelDTO1 = + ModelDTO.builder().withName("model_test").withLatestVersion(0).withAudit(audit).build(); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(modelDTO1); + ModelDTO deserModelDTO1 = JsonUtils.objectMapper().readValue(serJson1, ModelDTO.class); + Assertions.assertEquals(modelDTO1, deserModelDTO1); + Assertions.assertNull(deserModelDTO1.comment()); + Assertions.assertNull(deserModelDTO1.properties()); + } + + @Test + public void testInvalidModelDTO() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().withName("model_test").withLatestVersion(-1).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().withName("model_test").withLatestVersion(0).build(); + }); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java b/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java new file mode 100644 index 00000000000..5251246c377 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelVersionDTO { + + @Test + public void testModelVersionSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + Map props = ImmutableMap.of("key", "value"); + + ModelVersionDTO modelVersionDTO = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {"alias1", "alias2"}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO); + ModelVersionDTO deserModelVersionDTO = + JsonUtils.objectMapper().readValue(serJson, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO, deserModelVersionDTO); + + // Test with null aliases + ModelVersionDTO modelVersionDTO1 = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO1); + ModelVersionDTO deserModelVersionDTO1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO1, deserModelVersionDTO1); + Assertions.assertNull(deserModelVersionDTO1.aliases()); + + // Test with empty aliases + ModelVersionDTO modelVersionDTO2 = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO2); + ModelVersionDTO deserModelVersionDTO2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO2, deserModelVersionDTO2); + Assertions.assertArrayEquals(new String[] {}, deserModelVersionDTO2.aliases()); + + // Test with null comment and properties + ModelVersionDTO modelVersionDTO3 = + ModelVersionDTO.builder().withVersion(0).withUri("uri").withAudit(audit).build(); + + String serJson3 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO3); + ModelVersionDTO deserModelVersionDTO3 = + JsonUtils.objectMapper().readValue(serJson3, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO3, deserModelVersionDTO3); + Assertions.assertNull(deserModelVersionDTO3.comment()); + Assertions.assertNull(deserModelVersionDTO3.properties()); + } + + @Test + public void testInvalidModelVersionDTO() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(-1).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).withUri("").build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).withUri("uri").build(); + }); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java index b4b7383a7ea..3b5221ff1a7 100644 --- a/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java @@ -57,19 +57,24 @@ public void testCatalogCreateRequestSerDe() throws JsonProcessingException { String serJson1 = JsonUtils.objectMapper().writeValueAsString(request1); CatalogCreateRequest deserRequest1 = JsonUtils.objectMapper().readValue(serJson1, CatalogCreateRequest.class); - Assertions.assertEquals( deserRequest1.getType().name().toLowerCase(Locale.ROOT), deserRequest1.getProvider()); Assertions.assertNull(deserRequest1.getComment()); Assertions.assertNull(deserRequest1.getProperties()); + String json = "{\"name\":\"catalog_test\",\"type\":\"model\"}"; + CatalogCreateRequest deserRequest2 = + JsonUtils.objectMapper().readValue(json, CatalogCreateRequest.class); + Assertions.assertEquals("model", deserRequest2.getProvider()); + // Test using null provider with catalog type doesn't support managed catalog - CatalogCreateRequest request2 = - new CatalogCreateRequest("catalog_test", Catalog.Type.RELATIONAL, null, null, null); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> new CatalogCreateRequest("catalog_test", Catalog.Type.RELATIONAL, null, null, null)); - String serJson2 = JsonUtils.objectMapper().writeValueAsString(request2); + String json1 = "{\"name\":\"catalog_test\",\"type\":\"relational\"}"; Assertions.assertThrows( JsonMappingException.class, - () -> JsonUtils.objectMapper().readValue(serJson2, CatalogCreateRequest.class)); + () -> JsonUtils.objectMapper().readValue(json1, CatalogCreateRequest.class)); } } diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java new file mode 100644 index 00000000000..09dbdb8c312 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelRegisterRequest { + + @Test + public void testModelRegisterRequestSerDe() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + ModelRegisterRequest req = new ModelRegisterRequest("model", "comment", props); + + String serJson = JsonUtils.objectMapper().writeValueAsString(req); + ModelRegisterRequest deserReq = + JsonUtils.objectMapper().readValue(serJson, ModelRegisterRequest.class); + Assertions.assertEquals(req, deserReq); + + // Test with null comment and properties + ModelRegisterRequest req1 = new ModelRegisterRequest("model", null, null); + String serJson1 = JsonUtils.objectMapper().writeValueAsString(req1); + ModelRegisterRequest deserReq1 = + JsonUtils.objectMapper().readValue(serJson1, ModelRegisterRequest.class); + Assertions.assertEquals(req1, deserReq1); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java new file mode 100644 index 00000000000..4c0df6d73e8 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelVersionLinkRequest { + + @Test + public void testModelVersionLinkRequestSerDe() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + ModelVersionLinkRequest request = + new ModelVersionLinkRequest("uri", new String[] {"alias1", "alias2"}, "comment", props); + + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + ModelVersionLinkRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request, deserRequest); + + // Test with null aliases + ModelVersionLinkRequest request1 = new ModelVersionLinkRequest("uri", null, "comment", props); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(request1); + ModelVersionLinkRequest deserRequest1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request1, deserRequest1); + Assertions.assertNull(deserRequest1.getAliases()); + + // Test with empty aliases + ModelVersionLinkRequest request2 = + new ModelVersionLinkRequest("uri", new String[] {}, "comment", props); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(request2); + ModelVersionLinkRequest deserRequest2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request2, deserRequest2); + Assertions.assertEquals(0, deserRequest2.getAliases().length); + + // Test with null comment and properties + ModelVersionLinkRequest request3 = + new ModelVersionLinkRequest("uri", new String[] {"alias1", "alias2"}, null, null); + + String serJson3 = JsonUtils.objectMapper().writeValueAsString(request3); + ModelVersionLinkRequest deserRequest3 = + JsonUtils.objectMapper().readValue(serJson3, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request3, deserRequest3); + Assertions.assertNull(deserRequest3.getComment()); + Assertions.assertNull(deserRequest3.getProperties()); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java index 5f947820222..57813c0bc64 100644 --- a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java @@ -26,8 +26,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.time.Instant; +import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.authorization.Privileges; @@ -41,6 +43,8 @@ import org.apache.gravitino.dto.authorization.RoleDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.authorization.UserDTO; +import org.apache.gravitino.dto.model.ModelDTO; +import org.apache.gravitino.dto.model.ModelVersionDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.TableDTO; import org.apache.gravitino.dto.rel.partitioning.Partitioning; @@ -390,4 +394,76 @@ void testFileLocationResponseException() { FileLocationResponse response = new FileLocationResponse(); assertThrows(IllegalArgumentException.class, () -> response.validate()); } + + @Test + void testModelResponse() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + ModelDTO modelDTO = + ModelDTO.builder() + .withName("model1") + .withLatestVersion(0) + .withComment("comment1") + .withProperties(props) + .withAudit(audit) + .build(); + + ModelResponse response = new ModelResponse(modelDTO); + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + ModelResponse deserResponse = JsonUtils.objectMapper().readValue(serJson, ModelResponse.class); + + assertEquals(response, deserResponse); + + ModelResponse response1 = new ModelResponse(); + assertThrows(IllegalArgumentException.class, response1::validate); + } + + @Test + void testModelVersionListResponse() throws JsonProcessingException { + ModelVersionListResponse response1 = new ModelVersionListResponse(new int[] {}); + assertDoesNotThrow(response1::validate); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(response1); + ModelVersionListResponse deserResponse1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionListResponse.class); + assertEquals(response1, deserResponse1); + assertArrayEquals(new int[] {}, deserResponse1.getVersions()); + + ModelVersionListResponse response2 = new ModelVersionListResponse(new int[] {1, 2}); + assertDoesNotThrow(response2::validate); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(response2); + ModelVersionListResponse deserResponse2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionListResponse.class); + assertEquals(response2, deserResponse2); + assertArrayEquals(new int[] {1, 2}, deserResponse2.getVersions()); + } + + @Test + void testModelVersionResponse() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + ModelVersionDTO modelVersionDTO = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {"alias1", "alias2"}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + ModelVersionResponse response = new ModelVersionResponse(modelVersionDTO); + response.validate(); // No exception thrown + + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + ModelVersionResponse deserResponse = + JsonUtils.objectMapper().readValue(serJson, ModelVersionResponse.class); + assertEquals(response, deserResponse); + + ModelVersionResponse response1 = new ModelVersionResponse(); + assertThrows(IllegalArgumentException.class, response1::validate); + } } diff --git a/docs/open-api/models.yaml b/docs/open-api/models.yaml new file mode 100644 index 00000000000..713a7037cd6 --- /dev/null +++ b/docs/open-api/models.yaml @@ -0,0 +1,561 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- + +paths: + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + + get: + tags: + - model + summary: List models + operationId: listModels + responses: + "200": + $ref: "./openapi.yaml#/components/responses/EntityListResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + post: + tags: + - model + summary: Register model + operationId: registerModel + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ModelRegisterRequest" + examples: + ModelRegisterRequest: + $ref: "#/components/examples/ModelRegisterRequest" + responses: + "200": + $ref: "#/components/responses/ModelResponse" + "409": + description: Conflict - The target model already exists + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + ModelAlreadyExistsErrorResponse: + $ref: "#/components/examples/ModelAlreadyExistsException" + "404": + description: Not Found - The schema does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + + get: + tags: + - model + summary: Get model + operationId: getModel + description: Returns the specified model object + responses: + "200": + $ref: "#/components/responses/ModelResponse" + "404": + description: Not Found - The target fileset does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchCatalogException: + $ref: "./catalogs.yaml#/components/examples/NoSuchCatalogException" + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model + operationId: deleteModel + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + post: + tags: + - model + summary: link model version + operationId: linkModelVersion + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ModelVersionLinkRequest" + examples: + ModelVersionLinkRequest: + $ref: "#/components/examples/ModelVersionLinkRequest" + responses: + "200": + $ref: "./openapi.yaml#/components/responses/BaseResponse" + "404": + description: Not Found - The target model does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "409": + description: Conflict - The model version aliases already exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + ModelVersionAliasesAlreadyExistException: + $ref: "#/components/examples/ModelVersionAliasesAlreadyExistException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + + get: + tags: + - model + summary: List model versions + operationId: listModelVersions + responses: + "200": + $ref: "#/components/responses/ModelVersionListResponse" + "404": + description: Not Found - The target model does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions/{version}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + - $ref: "#/components/parameters/version" + + get: + tags: + - model + summary: Get model version + operationId: getModelVersion + responses: + "200": + $ref: "#/components/responses/ModelVersionResponse" + "404": + description: Not Found - The target model version does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelVersionException: + $ref: "#/components/examples/NoSuchModelVersionException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model version + operationId: deleteModelVersion + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/aliases/{alias}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + - $ref: "#/components/parameters/alias" + + get: + tags: + - model + summary: Get model version by alias + operationId: getModelVersionByAlias + responses: + "200": + $ref: "#/components/responses/ModelVersionResponse" + "404": + description: Not Found - The target model version does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelVersionException: + $ref: "#/components/examples/NoSuchModelVersionException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model version by alias + operationId: deleteModelVersionByAlias + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + parameters: + version: + name: version + in: path + required: true + description: The version of the model + schema: + type: integer + alias: + name: alias + in: path + required: true + description: The alias of the model version + schema: + type: string + + schemas: + Model: + type: object + required: + - name + - audit + - latestVersion + properties: + name: + type: string + description: The name of the model + latestVersion: + type: integer + description: The latest version of the model + comment: + type: string + description: The comment of the fileset + nullable: true + properties: + type: object + description: The properties of the fileset + nullable: true + default: {} + additionalProperties: + type: string + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + ModelVersion: + type: object + required: + - uri + - version + - audit + properties: + uri: + type: string + description: The uri of the model version + version: + type: integer + description: The version of the model + aliases: + type: array + description: The aliases of the model version + nullable: true + items: + type: string + comment: + type: string + description: The comment of the model version + nullable: true + properties: + type: object + description: The properties of the model version + nullable: true + default: {} + additionalProperties: + type: string + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + ModelRegisterRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the model. Can not be empty. + comment: + type: string + description: The comment of the model. Can be empty. + nullable: true + properties: + type: object + description: The properties of the model. Can be empty. + nullable: true + default: {} + additionalProperties: + type: string + + ModelVersionLinkRequest: + type: object + required: + - uri + properties: + uri: + type: string + description: The uri of the model version + aliases: + type: array + description: The aliases of the model version + nullable: true + items: + type: string + comment: + type: string + description: The comment of the model version + nullable: true + properties: + type: object + description: The properties of the model version + nullable: true + default: {} + additionalProperties: + type: string + + responses: + ModelResponse: + description: The response of model object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + model: + $ref: "#/components/schemas/Model" + examples: + ModelResponse: + $ref: "#/components/examples/ModelResponse" + ModelVersionListResponse: + description: The response of model version list + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + versions: + type: array + description: The list of model versions + items: + format: int32 + examples: + ModelVersionListResponse: + $ref: "#/components/examples/ModelVersionListResponse" + ModelVersionResponse: + description: The response of model version object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + modelVersion: + $ref: "#/components/schemas/ModelVersion" + examples: + ModelResponse: + $ref: "#/components/examples/ModelVersionResponse" + + examples: + ModelRegisterRequest: + value: { + "name": "model1", + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + } + } + + ModelVersionLinkRequest: + value: { + "uri": "hdfs://path/to/model", + "aliases": ["alias1", "alias2"], + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + } + } + + ModelResponse: + value: { + "code": 0, + "model" : { + "name": "model1", + "latestVersion": 0, + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + }, + "audit": { + "creator": "user1", + "createTime": "2021-01-01T00:00:00Z", + "lastModifier": "user1", + "lastModifiedTime": "2021-01-01T00:00:00Z" + } + } + } + + ModelVersionListResponse: + value: { + "code": 0, + "versions": [0, 1, 2] + } + + ModelVersionResponse: + value: { + "code": 0, + "modelVersion" : { + "uri": "hdfs://path/to/model", + "version": 0, + "aliases": ["alias1", "alias2"], + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + }, + "audit": { + "creator": "user1", + "createTime": "2021-01-01T00:00:00Z", + "lastModifier": "user1", + "lastModifiedTime": "2021-01-01T00:00:00Z" + } + } + } + + ModelAlreadyExistsException: + value: { + "code": 1004, + "type": "ModelAlreadyExistsException", + "message": "Model already exists", + "stack": [ + "org.apache.gravitino.exceptions.ModelAlreadyExistsException: Model already exists" + ] + } + + NoSuchModelException: + value: { + "code": 1003, + "type": "NoSuchModelException", + "message": "Model does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchModelException: Model does not exist" + ] + } + + ModelVersionAliasesAlreadyExistException: + value: { + "code": 1004, + "type": "ModelVersionAliasesAlreadyExistException", + "message": "Model version aliases already exist", + "stack": [ + "org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException: Model version aliases already exist" + ] + } + + NoSuchModelVersionException: + value: { + "code": 1003, + "type": "NoSuchModelVersionException", + "message": "Model version does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchModelVersionException: Model version does not exist" + ] + } diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index dd0564a7f9c..d0c941ab471 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -113,6 +113,20 @@ paths: /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics/{topic}: $ref: "./topics.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1topics~1%7Btopic%7D" + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1versions" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions/{version}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1versions~1%7Bversion%7D" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/aliases/{alias}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1aliases~1%7Balias%7D" /metalakes/{metalake}/users: $ref: "./users.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1users" @@ -430,6 +444,14 @@ components: schema: type: string + model: + name: model + in: path + description: The name of the model + required: true + schema: + type: string + tag: name: tag in: path @@ -476,6 +498,7 @@ components: - "COLUMN" - "FILESET" - "TOPIC" + - "MODEL" - "ROLE" metadataObjectFullName: name: metadataObjectFullName diff --git a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java index 63e53aefd59..2afc65482b3 100644 --- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java @@ -27,6 +27,7 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.catalog.ModelDispatcher; import org.apache.gravitino.catalog.PartitionDispatcher; import org.apache.gravitino.catalog.SchemaDispatcher; import org.apache.gravitino.catalog.TableDispatcher; @@ -118,6 +119,7 @@ protected void configure() { bind(gravitinoEnv.credentialOperationDispatcher()) .to(CredentialOperationDispatcher.class) .ranked(1); + bind(gravitinoEnv.modelDispatcher()).to(ModelDispatcher.class).ranked(1); } }); register(JsonProcessingExceptionMapper.class); diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index faf94f50648..b71219b0453 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -32,6 +32,8 @@ import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.MetalakeNotInUseException; +import org.apache.gravitino.exceptions.ModelAlreadyExistsException; +import org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NonEmptyCatalogException; import org.apache.gravitino.exceptions.NonEmptyMetalakeException; @@ -126,6 +128,11 @@ public static Response handleCredentialException( return CredentialExceptionHandler.INSTANCE.handle(op, metadataObjectName, "", e); } + public static Response handleModelException( + OperationType op, String model, String schema, Exception e) { + return ModelExceptionHandler.INSTANCE.handle(op, model, schema, e); + } + public static Response handleTestConnectionException(Exception e) { ErrorResponse response; if (e instanceof IllegalArgumentException) { @@ -729,6 +736,44 @@ public Response handle(OperationType op, String name, String parent, Exception e } } + private static class ModelExceptionHandler extends BaseExceptionHandler { + private static final ExceptionHandler INSTANCE = new ModelExceptionHandler(); + + private static String getModelErrorMsg( + String model, String operation, String schema, String reason) { + return String.format( + "Failed to operate model(s)%s operation [%s] under schema [%s], reason [%s]", + model, operation, schema, reason); + } + + @Override + public Response handle(OperationType op, String model, String schema, Exception e) { + String formatted = StringUtil.isBlank(model) ? "" : " [" + model + "]"; + String errorMsg = getModelErrorMsg(formatted, op.name(), schema, getErrorMsg(e)); + LOG.warn(errorMsg, e); + + if (e instanceof IllegalArgumentException) { + return Utils.illegalArguments(errorMsg, e); + + } else if (e instanceof NotFoundException) { + return Utils.notFound(errorMsg, e); + + } else if (e instanceof ModelAlreadyExistsException + || e instanceof ModelVersionAliasesAlreadyExistException) { + return Utils.alreadyExists(errorMsg, e); + + } else if (e instanceof ForbiddenException) { + return Utils.forbidden(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + + } else { + return super.handle(op, model, schema, e); + } + } + } + @VisibleForTesting static class BaseExceptionHandler extends ExceptionHandler { diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java new file mode 100644 index 00000000000..fd507821086 --- /dev/null +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.ModelDispatcher; +import org.apache.gravitino.dto.requests.ModelRegisterRequest; +import org.apache.gravitino.dto.requests.ModelVersionLinkRequest; +import org.apache.gravitino.dto.responses.BaseResponse; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.ModelResponse; +import org.apache.gravitino.dto.responses.ModelVersionListResponse; +import org.apache.gravitino.dto.responses.ModelVersionResponse; +import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; +import org.apache.gravitino.server.web.Utils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Path("metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models") +public class ModelOperations { + + private static final Logger LOG = LoggerFactory.getLogger(ModelOperations.class); + + private final ModelDispatcher modelDispatcher; + + @Context private HttpServletRequest httpRequest; + + @Inject + public ModelOperations(ModelDispatcher modelDispatcher) { + this.modelDispatcher = modelDispatcher; + } + + @GET + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-model", absolute = true) + public Response listModels( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema) { + LOG.info("Received list models request for schema: {}.{}.{}", metalake, catalog, schema); + Namespace modelNs = NamespaceUtil.ofModel(metalake, catalog, schema); + + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier[] modelIds = modelDispatcher.listModels(modelNs); + modelIds = modelIds == null ? new NameIdentifier[0] : modelIds; + LOG.info("List {} models under schema {}", modelIds.length, modelNs); + return Utils.ok(new EntityListResponse(modelIds)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LIST, "", schema, e); + } + } + + @GET + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model", absolute = true) + public Response getModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received get model request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + Model m = modelDispatcher.getModel(modelId); + LOG.info("Model got: {}", modelId); + return Utils.ok(new ModelResponse(DTOConverters.toDTO(m))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.GET, model, schema, e); + } + } + + @POST + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "register-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "register-model", absolute = true) + public Response registerModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + ModelRegisterRequest request) { + LOG.info( + "Received register model request: {}.{}.{}.{}", + metalake, + catalog, + schema, + request.getName()); + + try { + request.validate(); + NameIdentifier modelId = + NameIdentifierUtil.ofModel(metalake, catalog, schema, request.getName()); + + return Utils.doAs( + httpRequest, + () -> { + Model m = + modelDispatcher.registerModel( + modelId, request.getComment(), request.getProperties()); + LOG.info("Model registered: {}", modelId); + return Utils.ok(new ModelResponse(DTOConverters.toDTO(m))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.REGISTER, request.getName(), schema, e); + } + } + + @DELETE + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model", absolute = true) + public Response deleteModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received delete model request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModel(modelId); + if (!deleted) { + LOG.warn("Cannot find to be deleted model {} under schema {}", model, schema); + } else { + LOG.info("Model deleted: {}", modelId); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.DELETE, model, schema, e); + } + } + + @GET + @Path("{model}/versions") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-model-versions." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-model-versions", absolute = true) + public Response listModelVersions( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received list model versions request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + int[] versions = modelDispatcher.listModelVersions(modelId); + versions = versions == null ? new int[0] : versions; + LOG.info("List {} versions of model {}", versions.length, modelId); + return Utils.ok(new ModelVersionListResponse(versions)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LIST_VERSIONS, model, schema, e); + } + } + + @GET + @Path("{model}/versions/{version}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model-version", absolute = true) + public Response getModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("version") int version) { + LOG.info( + "Received get model version request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + version); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + ModelVersion mv = modelDispatcher.getModelVersion(modelId, version); + LOG.info("Model version got: {}.{}", modelId, version); + return Utils.ok(new ModelVersionResponse(DTOConverters.toDTO(mv))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.GET, versionString(model, version), schema, e); + } + } + + @GET + @Path("{model}/aliases/{alias}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model-alias." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model-alias", absolute = true) + public Response getModelVersionByAlias( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("alias") String alias) { + LOG.info( + "Received get model version alias request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + alias); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + ModelVersion mv = modelDispatcher.getModelVersion(modelId, alias); + LOG.info("Model version alias got: {}.{}", modelId, alias); + return Utils.ok(new ModelVersionResponse(DTOConverters.toDTO(mv))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.GET, aliasString(model, alias), schema, e); + } + } + + @POST + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "link-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "link-model-version", absolute = true) + public Response linkModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + ModelVersionLinkRequest request) { + LOG.info("Received link model version request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + request.validate(); + + return Utils.doAs( + httpRequest, + () -> { + modelDispatcher.linkModelVersion( + modelId, + request.getUri(), + request.getAliases(), + request.getComment(), + request.getProperties()); + LOG.info("Model version linked: {}", modelId); + return Utils.ok(new BaseResponse()); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LINK, model, schema, e); + } + } + + @DELETE + @Path("{model}/versions/{version}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model-version", absolute = true) + public Response deleteModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("version") int version) { + LOG.info( + "Received delete model version request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + version); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModelVersion(modelId, version); + if (!deleted) { + LOG.warn("Cannot find to be deleted version {} in model {}", version, model); + } else { + LOG.info("Model version deleted: {}.{}", modelId, version); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.DELETE, versionString(model, version), schema, e); + } + } + + @DELETE + @Path("{model}/aliases/{alias}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model-alias." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model-alias", absolute = true) + public Response deleteModelVersionByAlias( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("alias") String alias) { + LOG.info( + "Received delete model version by alias request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + alias); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModelVersion(modelId, alias); + if (!deleted) { + LOG.warn( + "Cannot find to be deleted model version by alias {} in model {}", alias, model); + } else { + LOG.info("Model version by alias deleted: {}.{}", modelId, alias); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.DELETE, aliasString(model, alias), schema, e); + } + } + + private String versionString(String model, int version) { + return model + " version(" + version + ")"; + } + + private String aliasString(String model, String alias) { + return model + " alias(" + alias + ")"; + } +} diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java index 8d4bc322ae7..2b8abd91f1d 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java @@ -35,4 +35,7 @@ public enum OperationType { REVOKE, ASSOCIATE, SET, + REGISTER, // An operation to register a model + LIST_VERSIONS, // An operation to list versions of a model + LINK // An operation to link a version to a model } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java new file mode 100644 index 00000000000..42e48d0302f --- /dev/null +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java @@ -0,0 +1,843 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.ModelDispatcher; +import org.apache.gravitino.dto.requests.ModelRegisterRequest; +import org.apache.gravitino.dto.requests.ModelVersionLinkRequest; +import org.apache.gravitino.dto.responses.BaseResponse; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.ErrorConstants; +import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.ModelResponse; +import org.apache.gravitino.dto.responses.ModelVersionListResponse; +import org.apache.gravitino.dto.responses.ModelVersionResponse; +import org.apache.gravitino.exceptions.ModelAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchModelException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; +import org.apache.gravitino.rest.RESTUtils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.glassfish.jersey.internal.inject.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.glassfish.jersey.test.TestProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelOperations extends JerseyTest { + + private static class MockServletRequestFactory extends ServletRequestFactoryBase { + + @Override + public HttpServletRequest get() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn(null); + return request; + } + } + + private ModelDispatcher modelDispatcher = mock(ModelDispatcher.class); + + private AuditInfo testAuditInfo = + AuditInfo.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + private Map properties = ImmutableMap.of("key1", "value"); + + private String metalake = "metalake_for_model_test"; + + private String catalog = "catalog_for_model_test"; + + private String schema = "schema_for_model_test"; + + private Namespace modelNs = NamespaceUtil.ofModel(metalake, catalog, schema); + + @Override + protected Application configure() { + try { + forceSet( + TestProperties.CONTAINER_PORT, String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(ModelOperations.class); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bind(modelDispatcher).to(ModelDispatcher.class).ranked(2); + bindFactory(TestModelOperations.MockServletRequestFactory.class) + .to(HttpServletRequest.class); + } + }); + + return resourceConfig; + } + + @Test + public void testListModels() { + NameIdentifier modelId1 = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + NameIdentifier modelId2 = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model2"); + NameIdentifier[] modelIds = new NameIdentifier[] {modelId1, modelId2}; + when(modelDispatcher.listModels(modelNs)).thenReturn(modelIds); + + Response response = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + EntityListResponse resp = response.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp.getCode()); + Assertions.assertArrayEquals(modelIds, resp.identifiers()); + + // Test mock return null for listModels + when(modelDispatcher.listModels(modelNs)).thenReturn(null); + Response resp1 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + EntityListResponse resp2 = resp1.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp2.getCode()); + Assertions.assertEquals(0, resp2.identifiers().length); + + // Test mock return empty array for listModels + when(modelDispatcher.listModels(modelNs)).thenReturn(new NameIdentifier[0]); + Response resp3 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + EntityListResponse resp4 = resp3.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp4.getCode()); + Assertions.assertEquals(0, resp4.identifiers().length); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")).when(modelDispatcher).listModels(modelNs); + Response resp5 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).listModels(modelNs); + Response resp6 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp6.getStatus()); + + ErrorResponse errorResp1 = resp6.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetModel() { + Model mockModel = mockModel("model1", "comment1", 0); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.getModel(modelId)).thenReturn(mockModel); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelResponse modelResp = resp.readEntity(ModelResponse.class); + Assertions.assertEquals(0, modelResp.getCode()); + + Model resultModel = modelResp.getModel(); + compare(mockModel, resultModel); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")).when(modelDispatcher).getModel(modelId); + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).getModel(modelId); + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testRegisterModel() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + Model mockModel = mockModel("model1", "comment1", 0); + when(modelDispatcher.registerModel(modelId, "comment1", properties)).thenReturn(mockModel); + + ModelRegisterRequest req = new ModelRegisterRequest("model1", "comment1", properties); + Response resp = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelResponse modelResp = resp.readEntity(ModelResponse.class); + Assertions.assertEquals(0, modelResp.getCode()); + compare(mockModel, modelResp.getModel()); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp1 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw ModelAlreadyExistsException + doThrow(new ModelAlreadyExistsException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp2 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp1.getCode()); + Assertions.assertEquals( + ModelAlreadyExistsException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp3 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testDeleteModel() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.deleteModel(modelId)).thenReturn(true); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + // Test mock return false for deleteModel + when(modelDispatcher.deleteModel(modelId)).thenReturn(false); + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).deleteModel(modelId); + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testListModelVersions() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + int[] versions = new int[] {0, 1, 2}; + when(modelDispatcher.listModelVersions(modelId)).thenReturn(versions); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelVersionListResponse versionListResp = resp.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp.getCode()); + Assertions.assertArrayEquals(versions, versionListResp.getVersions()); + + // Test mock return null for listModelVersions + when(modelDispatcher.listModelVersions(modelId)).thenReturn(null); + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + ModelVersionListResponse versionListResp1 = resp1.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp1.getCode()); + Assertions.assertEquals(0, versionListResp1.getVersions().length); + + // Test mock return empty array for listModelVersions + when(modelDispatcher.listModelVersions(modelId)).thenReturn(new int[0]); + Response resp2 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp2.getMediaType()); + + ModelVersionListResponse versionListResp2 = resp2.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp2.getCode()); + Assertions.assertEquals(0, versionListResp2.getVersions().length); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .listModelVersions(modelId); + Response resp3 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).listModelVersions(modelId); + Response resp4 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp1 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + ModelVersion mockModelVersion = + mockModelVersion(0, "uri1", new String[] {"alias1"}, "comment1"); + when(modelDispatcher.getModelVersion(modelIdent, 0)).thenReturn(mockModelVersion); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelVersionResponse versionResp = resp.readEntity(ModelVersionResponse.class); + Assertions.assertEquals(0, versionResp.getCode()); + compare(mockModelVersion, versionResp.getModelVersion()); + + // Test mock throw NoSuchModelVersionException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, 0); + + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, 0); + + Response resp2 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + + // Test get model version by alias + when(modelDispatcher.getModelVersion(modelIdent, "alias1")).thenReturn(mockModelVersion); + + Response resp3 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + ModelVersionResponse versionResp1 = resp3.readEntity(ModelVersionResponse.class); + Assertions.assertEquals(0, versionResp1.getCode()); + compare(mockModelVersion, versionResp1.getModelVersion()); + + // Test mock throw NoSuchModelVersionException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, "alias1"); + + Response resp4 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp2 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp2.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp2.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, "alias1"); + + Response resp5 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp3 = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp3.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp3.getType()); + } + + @Test + public void testLinkModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + doNothing() + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + ModelVersionLinkRequest req = + new ModelVersionLinkRequest("uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + BaseResponse baseResponse = resp.readEntity(BaseResponse.class); + Assertions.assertEquals(0, baseResponse.getCode()); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw ModelVersionAliasesAlreadyExistException + doThrow(new ModelAlreadyExistsException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp1.getCode()); + Assertions.assertEquals( + ModelAlreadyExistsException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp3 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testDeleteModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.deleteModelVersion(modelIdent, 0)).thenReturn(true); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + // Test mock return false for deleteModelVersion + when(modelDispatcher.deleteModelVersion(modelIdent, 0)).thenReturn(false); + + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test mock return true for deleteModelVersion using alias + when(modelDispatcher.deleteModelVersion(modelIdent, "alias1")).thenReturn(true); + + Response resp2 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp2.getMediaType()); + + DropResponse dropResp2 = resp2.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp2.getCode()); + + // Test mock return false for deleteModelVersion using alias + when(modelDispatcher.deleteModelVersion(modelIdent, "alias1")).thenReturn(false); + + Response resp3 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + DropResponse dropResp3 = resp3.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp3.getCode()); + Assertions.assertFalse(dropResp3.dropped()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .deleteModelVersion(modelIdent, 0); + + Response resp4 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp1 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException using alias + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .deleteModelVersion(modelIdent, "alias1"); + + Response resp5 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp2 = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + private String modelPath() { + return "/metalakes/" + metalake + "/catalogs/" + catalog + "/schemas/" + schema + "/models"; + } + + private Model mockModel(String modelName, String comment, int latestVersion) { + Model mockModel = mock(Model.class); + when(mockModel.name()).thenReturn(modelName); + when(mockModel.comment()).thenReturn(comment); + when(mockModel.latestVersion()).thenReturn(latestVersion); + when(mockModel.properties()).thenReturn(properties); + when(mockModel.auditInfo()).thenReturn(testAuditInfo); + return mockModel; + } + + private ModelVersion mockModelVersion(int version, String uri, String[] aliases, String comment) { + ModelVersion mockModelVersion = mock(ModelVersion.class); + when(mockModelVersion.version()).thenReturn(version); + when(mockModelVersion.uri()).thenReturn(uri); + when(mockModelVersion.aliases()).thenReturn(aliases); + when(mockModelVersion.comment()).thenReturn(comment); + when(mockModelVersion.properties()).thenReturn(properties); + when(mockModelVersion.auditInfo()).thenReturn(testAuditInfo); + return mockModelVersion; + } + + private void compare(Model left, Model right) { + Assertions.assertEquals(left.name(), right.name()); + Assertions.assertEquals(left.comment(), right.comment()); + Assertions.assertEquals(left.properties(), right.properties()); + + Assertions.assertNotNull(right.auditInfo()); + Assertions.assertEquals(left.auditInfo().creator(), right.auditInfo().creator()); + Assertions.assertEquals(left.auditInfo().createTime(), right.auditInfo().createTime()); + Assertions.assertEquals(left.auditInfo().lastModifier(), right.auditInfo().lastModifier()); + Assertions.assertEquals( + left.auditInfo().lastModifiedTime(), right.auditInfo().lastModifiedTime()); + } + + private void compare(ModelVersion left, ModelVersion right) { + Assertions.assertEquals(left.version(), right.version()); + Assertions.assertEquals(left.uri(), right.uri()); + Assertions.assertArrayEquals(left.aliases(), right.aliases()); + Assertions.assertEquals(left.comment(), right.comment()); + Assertions.assertEquals(left.properties(), right.properties()); + + Assertions.assertNotNull(right.auditInfo()); + Assertions.assertEquals(left.auditInfo().creator(), right.auditInfo().creator()); + Assertions.assertEquals(left.auditInfo().createTime(), right.auditInfo().createTime()); + Assertions.assertEquals(left.auditInfo().lastModifier(), right.auditInfo().lastModifier()); + Assertions.assertEquals( + left.auditInfo().lastModifiedTime(), right.auditInfo().lastModifiedTime()); + } +} From 061f24bcae2e1ff265b032eb04e57682884ee5a9 Mon Sep 17 00:00:00 2001 From: roryqi Date: Thu, 26 Dec 2024 15:50:06 +0800 Subject: [PATCH 7/7] [#5993] refactor: Move the JdbcAuthorizationPlugin to authorization-common module (#5994) ### What changes were proposed in this pull request? Move the JdbcAuthorizationPlugin to authorization-common module ### Why are the changes needed? Fix: #5993 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Just refactor. --- .../authorization-common/build.gradle.kts | 1 + .../common/AuthorizationProperties.java | 4 +- .../jdbc/JdbcAuthorizationPlugin.java | 3 +- .../JdbcAuthorizationProperties.java | 5 +- .../jdbc/JdbcAuthorizationSQL.java | 2 +- .../jdbc/JdbcMetadataObject.java | 0 .../authorization/jdbc/JdbcPrivilege.java | 0 .../jdbc/JdbcSecurableObject.java | 0 .../JdbcSecurableObjectMappingProvider.java | 0 .../jdbc/TestJdbcAuthorizationPlugin.java} | 3 +- .../authorization-jdbc/build.gradle.kts | 96 ------------------- settings.gradle.kts | 2 +- 12 files changed, 10 insertions(+), 106 deletions(-) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java (98%) rename authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/{common => jdbc}/JdbcAuthorizationProperties.java (92%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java (99%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java (100%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java (100%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java (100%) rename authorizations/{authorization-jdbc => authorization-common}/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java (100%) rename authorizations/{authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java => authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java} (98%) delete mode 100644 authorizations/authorization-jdbc/build.gradle.kts diff --git a/authorizations/authorization-common/build.gradle.kts b/authorizations/authorization-common/build.gradle.kts index ba64510f2ce..9bab92dac3e 100644 --- a/authorizations/authorization-common/build.gradle.kts +++ b/authorizations/authorization-common/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) + implementation(libs.commons.dbcp2) implementation(libs.guava) implementation(libs.javax.jaxb.api) { exclude("*") diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java index 3005cc5f3e9..3ece6353d6e 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java @@ -31,9 +31,9 @@ public AuthorizationProperties(Map properties) { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - abstract String getPropertiesPrefix(); + public abstract String getPropertiesPrefix(); - abstract void validate(); + public abstract void validate(); public static void validate(String type, Map properties) { switch (type) { diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java similarity index 98% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java index d9bc28636c3..cc3190413e1 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -40,7 +40,6 @@ import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; -import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.meta.AuditInfo; @@ -55,7 +54,7 @@ * JDBC-based authorization plugins can inherit this class and implement their own SQL statements. */ @Unstable -abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL { +public abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL { private static final String GROUP_PREFIX = "GRAVITINO_GROUP_"; private static final Logger LOG = LoggerFactory.getLogger(JdbcAuthorizationPlugin.class); diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java similarity index 92% rename from authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java index 9a5e7c6cc97..69a12135023 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.common; +package org.apache.gravitino.authorization.jdbc; import java.util.Map; +import org.apache.gravitino.authorization.common.AuthorizationProperties; /** The properties for JDBC authorization plugin. */ public class JdbcAuthorizationProperties extends AuthorizationProperties { @@ -39,7 +40,7 @@ private void check(String key, String errorMsg) { } @Override - String getPropertiesPrefix() { + public String getPropertiesPrefix() { return CONFIG_PREFIX; } diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java similarity index 99% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java index f7171ff354a..de031f70e78 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java @@ -25,7 +25,7 @@ /** Interface for SQL operations of the underlying access control system. */ @Unstable -interface JdbcAuthorizationSQL { +public interface JdbcAuthorizationSQL { /** * Get SQL statements for creating a user. diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java diff --git a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java similarity index 98% rename from authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java rename to authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java index e261fad78d2..ab91ba81e93 100644 --- a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java @@ -34,7 +34,6 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.User; -import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; import org.apache.gravitino.meta.RoleEntity; @@ -42,7 +41,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class JdbcAuthorizationPluginTest { +public class TestJdbcAuthorizationPlugin { private static List expectSQLs = Lists.newArrayList(); private static List expectTypes = Lists.newArrayList(); private static List expectObjectNames = Lists.newArrayList(); diff --git a/authorizations/authorization-jdbc/build.gradle.kts b/authorizations/authorization-jdbc/build.gradle.kts deleted file mode 100644 index 1a61f7c0cf9..00000000000 --- a/authorizations/authorization-jdbc/build.gradle.kts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -description = "authorization-jdbc" - -plugins { - `maven-publish` - id("java") - id("idea") -} - -dependencies { - implementation(project(":api")) { - exclude(group = "*") - } - implementation(project(":core")) { - exclude(group = "*") - } - implementation(project(":authorizations:authorization-common")) { - exclude(group = "*") - } - implementation(libs.bundles.log4j) - implementation(libs.commons.lang3) - implementation(libs.guava) - implementation(libs.javax.jaxb.api) { - exclude("*") - } - implementation(libs.javax.ws.rs.api) - implementation(libs.jettison) - compileOnly(libs.lombok) - implementation(libs.mail) - implementation(libs.rome) - implementation(libs.commons.dbcp2) - - testImplementation(project(":common")) - testImplementation(project(":clients:client-java")) - testImplementation(project(":server")) - testImplementation(project(":catalogs:catalog-common")) - testImplementation(project(":integration-test-common", "testArtifacts")) - testImplementation(libs.junit.jupiter.api) - testImplementation(libs.mockito.core) - testImplementation(libs.testcontainers) - testRuntimeOnly(libs.junit.jupiter.engine) -} - -tasks { - val runtimeJars by registering(Copy::class) { - from(configurations.runtimeClasspath) - into("build/libs") - } - - val copyAuthorizationLibs by registering(Copy::class) { - dependsOn("jar", runtimeJars) - from("build/libs") { - exclude("guava-*.jar") - exclude("log4j-*.jar") - exclude("slf4j-*.jar") - } - into("$rootDir/distribution/package/authorizations/ranger/libs") - } - - register("copyLibAndConfig", Copy::class) { - dependsOn(copyAuthorizationLibs) - } - - jar { - dependsOn(runtimeJars) - } -} - -tasks.test { - dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars") - - val skipITs = project.hasProperty("skipITs") - if (skipITs) { - // Exclude integration tests - exclude("**/integration/test/**") - } else { - dependsOn(tasks.jar) - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index f38443db206..562614764b3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,7 @@ if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() == true) } include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") -include("authorizations:authorization-ranger", "authorizations:authorization-jdbc", "authorizations:authorization-common", "authorizations:authorization-chain") +include("authorizations:authorization-ranger", "authorizations:authorization-common", "authorizations:authorization-chain") include("trino-connector:trino-connector", "trino-connector:integration-test") include("spark-connector:spark-common") // kyuubi hive connector doesn't support 2.13 for Spark3.3