diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 2464a06e1b1..1b973f2ad22 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -108,8 +108,7 @@ jobs: iceberg/iceberg-rest-server/build/*.log integration-test/build/*.log integration-test/build/*.tar - integration-test/build/trino-ci-container-log/hive/*.* - integration-test/build/trino-ci-container-log/hdfs/*.* + integration-test/build/trino-ci-container-log distribution/package/logs/*.out distribution/package/logs/*.log catalogs/**/*.log diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cbb9eaffb68..0cf17683a0d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,6 +68,39 @@ jobs: - name: Build with Gradle run: ./gradlew build -x test -PjdkVersion=8 + # To check the spark-connector is compatible with scala2.13 + spark-connector-build: + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: changes + if: needs.changes.outputs.source_changes == 'true' + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: 'temurin' + cache: 'gradle' + + - name: Free up disk space + run: | + dev/ci/util_free_space.sh + + - name: Build with Scala2.13 + run: | + ./gradlew :spark-connector:spark-3.4:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false + ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false + + - name: Upload unit tests report + uses: actions/upload-artifact@v3 + if: failure() + with: + name: unit test report + path: | + build/reports + spark-connector/**/*.log + build: # The type of runner that the job will run on runs-on: ubuntu-latest diff --git a/.github/workflows/cron-integration-test.yml b/.github/workflows/cron-integration-test.yml index 8ae27638473..1e996aeb24d 100644 --- a/.github/workflows/cron-integration-test.yml +++ b/.github/workflows/cron-integration-test.yml @@ -96,8 +96,7 @@ jobs: iceberg/iceberg-rest-server/build/*.log integration-test/build/*.log integration-test/build/*.tar - integration-test/build/trino-ci-container-log/hive/*.* - integration-test/build/trino-ci-container-log/hdfs/*.* + integration-test/build/trino-ci-container-log distribution/package/logs/*.out distribution/package/logs/*.log catalogs/**/*.log diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 1993a209a4b..f064ea789af 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -15,6 +15,7 @@ on: - 'gravitino-ci-trino' - 'gravitino-ci-doris' - 'gravitino-ci-ranger' + - 'gravitino-iceberg-rest-server' - 'trino' - 'hive' - 'ranger' @@ -64,6 +65,9 @@ jobs: elif [ "${{ github.event.inputs.image }}" == "ranger" ]; then echo "image_type=ranger" >> $GITHUB_ENV echo "image_name=datastrato/ranger" >> $GITHUB_ENV + elif [ "${{ github.event.inputs.image }}" == "gravitino-iceberg-rest-server" ]; then + echo "image_type=iceberg-rest-server" >> $GITHUB_ENV + echo "image_name=datastrato/gravitino-iceberg-rest-server" >> $GITHUB_ENV fi - name: Check publish Docker token diff --git a/api/src/main/java/org/apache/gravitino/rel/TableChange.java b/api/src/main/java/org/apache/gravitino/rel/TableChange.java index a24f45adabe..ea5849eb6ef 100644 --- a/api/src/main/java/org/apache/gravitino/rel/TableChange.java +++ b/api/src/main/java/org/apache/gravitino/rel/TableChange.java @@ -20,6 +20,7 @@ package org.apache.gravitino.rel; +import com.google.common.base.Preconditions; import java.util.Arrays; import java.util.Objects; import org.apache.gravitino.annotation.Evolving; @@ -1353,6 +1354,8 @@ final class UpdateColumnPosition implements ColumnChange { private final ColumnPosition position; private UpdateColumnPosition(String[] fieldName, ColumnPosition position) { + Preconditions.checkArgument( + position != ColumnPosition.defaultPos(), "Position cannot be DEFAULT"); this.fieldName = fieldName; this.position = position; } diff --git a/api/src/test/java/org/apache/gravitino/TestTableChange.java b/api/src/test/java/org/apache/gravitino/TestTableChange.java index 2238fc01ad6..8f32c238f77 100644 --- a/api/src/test/java/org/apache/gravitino/TestTableChange.java +++ b/api/src/test/java/org/apache/gravitino/TestTableChange.java @@ -43,6 +43,7 @@ import org.apache.gravitino.rel.expressions.literals.Literals; import org.apache.gravitino.rel.types.Type; import org.apache.gravitino.rel.types.Types; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class TestTableChange { @@ -168,6 +169,17 @@ public void testUpdateNestedColumnPosition() { assertEquals(newPosition, updateColumnPosition.getPosition()); } + @Test + public void testDoesNotAllowDefaultInColumnPosition() { + String[] fieldName = {"Name", "Last Name"}; + ColumnPosition newPosition = TableChange.ColumnPosition.defaultPos(); + Exception exception = + Assertions.assertThrowsExactly( + IllegalArgumentException.class, + () -> TableChange.updateColumnPosition(fieldName, newPosition)); + assertEquals("Position cannot be DEFAULT", exception.getMessage()); + } + @Test public void testUpdateColumnComment() { String[] fieldName = {"First Name"}; diff --git a/bin/common.sh b/bin/common.sh index f067750800d..a6f002ad91d 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -105,3 +105,20 @@ if [[ -n "${JAVA_HOME}" ]]; then else export JAVA_RUNNER=java fi + +function printArt() { + local lineLength=70 + local versionText="Version: ${GRAVITINO_VERSION} " + local versionTextLength=${#versionText} + local paddingLength=$((lineLength - versionTextLength - 3)) + local versionLine=$(printf "#%${paddingLength}s%s#" "" "$versionText") + + echo "#####################################################################" + echo "# ____ ____ _ ___ ___ ___ _____ ___ _ _ ___ #" + echo "# / ___| | _ \ / \ \ \ / / |_ _||_ _||_ _|| \ | | / _ \ #" + echo "# | | __ | |_) | / _ \ \ \ / / | | | | | | | \| || | | | #" + echo "# | |_| | | _ < / ___ \ \ V / | | | | | | | |\ || |_| | #" + echo "# \____| |_| \_\/_/ \_\ \_/ |___| |_| |___||_| \_| \___/ #" + echo "$versionLine" + echo "#####################################################################" +} diff --git a/bin/gravitino-iceberg-rest-server.sh b/bin/gravitino-iceberg-rest-server.sh index d0b3349bb52..61f0080123d 100755 --- a/bin/gravitino-iceberg-rest-server.sh +++ b/bin/gravitino-iceberg-rest-server.sh @@ -47,6 +47,7 @@ function check_process_status() { if [[ -z "${pid}" ]]; then echo "GravitinoIcebergRESTServer is not running" else + printArt echo "GravitinoIcebergRESTServer is running[PID:$pid]" fi } @@ -116,8 +117,6 @@ function start() { if [[ -z "${pid}" ]]; then echo "GravitinoIcebergRESTServer start error!" return 1; - else - echo "GravitinoIcebergRESTServer start success!" fi sleep 2 diff --git a/bin/gravitino.sh b/bin/gravitino.sh index e29d05c0381..a19d73d8733 100755 --- a/bin/gravitino.sh +++ b/bin/gravitino.sh @@ -47,6 +47,7 @@ function check_process_status() { if [[ -z "${pid}" ]]; then echo "Gravitino Server is not running" else + printArt echo "Gravitino Server is running[PID:$pid]" fi } @@ -116,8 +117,6 @@ function start() { if [[ -z "${pid}" ]]; then echo "Gravitino Server start error!" return 1; - else - echo "Gravitino Server start success!" fi sleep 2 diff --git a/build.gradle.kts b/build.gradle.kts index 13023bcd83e..6943c5f96e0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -492,7 +492,7 @@ tasks.rat { "ROADMAP.md", "clients/client-python/.pytest_cache/*", "clients/client-python/.venv/*", - "clients/client-python/gravitino.egg-info/*", + "clients/client-python/apache_gravitino.egg-info/*", "clients/client-python/gravitino/utils/exceptions.py", "clients/client-python/gravitino/utils/http_client.py", "clients/client-python/tests/unittests/htmlcov/*", @@ -545,6 +545,13 @@ tasks { rename { fileName -> fileName.replace(".template", "") } + eachFile { + if (name == "gravitino-env.sh") { + filter { line -> + line.replace("GRAVITINO_VERSION_PLACEHOLDER", "$version") + } + } + } fileMode = 0b111101101 } copy { @@ -571,7 +578,7 @@ tasks { doLast { copy { from(projectDir.dir("conf")) { - include("${rootProject.name}-iceberg-rest-server.conf.template", "log4j2.properties.template") + include("${rootProject.name}-iceberg-rest-server.conf.template", "${rootProject.name}-env.sh.template", "log4j2.properties.template") into("${rootProject.name}-iceberg-rest-server/conf") } from(projectDir.dir("bin")) { @@ -582,6 +589,13 @@ tasks { rename { fileName -> fileName.replace(".template", "") } + eachFile { + if (name == "gravitino-env.sh") { + filter { line -> + line.replace("GRAVITINO_VERSION_PLACEHOLDER", "$version") + } + } + } fileMode = 0b111101101 } diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java index 4a1f46fd670..6e4aae37ab1 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java @@ -63,4 +63,8 @@ public class IcebergConstants { public static final String ICEBERG_METRICS_QUEUE_CAPACITY = "metricsQueueCapacity"; public static final String GRAVITINO_ICEBERG_REST_SERVICE_NAME = "iceberg-rest"; + + public static final String ICEBERG_REST_CATALOG_PROVIDER = "catalog-provider"; + + public static final String GRAVITINO_DEFAULT_CATALOG = "__gravitino_default_catalog"; } diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index ba85c3b6e2f..03d381e190c 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -94,6 +94,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } register("copyLibAndConfig", Copy::class) { diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java index b8f31699d5e..4ea2fbdd9db 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java @@ -65,9 +65,8 @@ public String login(String keytabFilePath) throws IOException { // Login UserGroupInformation.setConfiguration(hadoopConf); - UserGroupInformation kerberosLoginUgi = - UserGroupInformation.loginUserFromKeytabAndReturnUGI(catalogPrincipal, keytabFilePath); - UserGroupInformation.setLoginUser(kerberosLoginUgi); + UserGroupInformation.loginUserFromKeytab(catalogPrincipal, keytabFilePath); + UserGroupInformation kerberosLoginUgi = UserGroupInformation.getLoginUser(); // Refresh the cache if it's out of date. if (refreshCredentials) { diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index c9c7422c640..93efe11bf65 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -157,6 +157,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } register("copyLibAndConfig", Copy::class) { diff --git a/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/JdbcCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/JdbcCatalogOperations.java index 4d2747a66ac..cbbbd469e37 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/JdbcCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/JdbcCatalogOperations.java @@ -22,6 +22,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.Driver; import java.sql.DriverManager; import java.sql.SQLException; @@ -54,6 +56,7 @@ import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.SupportsSchemas; +import org.apache.gravitino.exceptions.GravitinoRuntimeException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NoSuchTableException; @@ -98,6 +101,20 @@ public class JdbcCatalogOperations implements CatalogOperations, SupportsSchemas private final JdbcColumnDefaultValueConverter columnDefaultValueConverter; + public static class JDBCDriverInfo { + public String name; + public String version; + public int majorVersion; + public int minorVersion; + + public JDBCDriverInfo(String driverName, String version, int majorVersion, int minorVersion) { + this.name = driverName; + this.version = version; + this.majorVersion = majorVersion; + this.minorVersion = minorVersion; + } + } + /** * Constructs a new instance of JdbcCatalogOperations. * @@ -148,6 +165,8 @@ public void initialize( JdbcConfig jdbcConfig = new JdbcConfig(resultConf); this.dataSource = DataSourceUtils.createDataSource(jdbcConfig); + + checkJDBCDriverVersion(); this.databaseOperation.initialize(dataSource, exceptionConverter, resultConf); this.tableOperation.initialize( dataSource, exceptionConverter, jdbcTypeConverter, columnDefaultValueConverter, resultConf); @@ -486,6 +505,35 @@ private Table renameTable(NameIdentifier tableIdent, TableChange.RenameTable ren return loadTable(NameIdentifier.of(tableIdent.namespace(), renameTable.getNewName())); } + /** + * Get the JDBC driver name and version + * + * @return Returns the JDBC driver info + */ + public JDBCDriverInfo getDiverInfo() { + try (Connection conn = dataSource.getConnection()) { + DatabaseMetaData metaData = conn.getMetaData(); + return new JDBCDriverInfo( + metaData.getDriverName(), + metaData.getDriverVersion(), + metaData.getDriverMajorVersion(), + metaData.getDriverMinorVersion()); + } catch (final SQLException se) { + throw new GravitinoRuntimeException( + se, "Failed to get JDBC driver information %s: ", se.getMessage()); + } + } + + /** + * Check the version of JDBC driver can supported. + * + * @return Returns the result of checking the jdbc driver version. If success return true, + * otherwise throw a RuntimeException + */ + public boolean checkJDBCDriverVersion() { + return true; + } + private Table internalAlterTable(NameIdentifier tableIdent, TableChange... changes) throws NoSuchTableException, IllegalArgumentException { String databaseName = NameIdentifier.of(tableIdent.namespace().levels()).name(); diff --git a/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java index 4253301e5f5..35f3a659c93 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java @@ -33,6 +33,8 @@ public class MySQLProtocolCompatibleCatalogOperations extends JdbcCatalogOperati private static final Logger LOG = LoggerFactory.getLogger(MySQLProtocolCompatibleCatalogOperations.class); + private static final int MYSQL_JDBC_DRIVER_MINIMAL_SUPPORT_VERSION = 8; + public MySQLProtocolCompatibleCatalogOperations( JdbcExceptionConverter exceptionConverter, JdbcTypeConverter jdbcTypeConverter, @@ -47,6 +49,18 @@ public MySQLProtocolCompatibleCatalogOperations( columnDefaultValueConverter); } + @Override + public boolean checkJDBCDriverVersion() { + JDBCDriverInfo driverInfo = getDiverInfo(); + if (driverInfo.majorVersion < MYSQL_JDBC_DRIVER_MINIMAL_SUPPORT_VERSION) { + throw new RuntimeException( + String.format( + "Mysql catalog does not support the jdbc driver version %s, minimal required version is 8.0", + driverInfo.version)); + } + return true; + } + @Override public void close() { super.close(); diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index 0d01a5cc03f..c10496067ae 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -74,6 +74,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } register("copyLibAndConfig", Copy::class) { diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java index 40cb3254bc6..ffbcb98d3c6 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java @@ -57,7 +57,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestDorisTablePartitionOperations extends TestDoris { private static final String databaseName = GravitinoITUtils.genRandomName("doris_test_db"); private static final Integer DEFAULT_BUCKET_SIZE = 1; diff --git a/catalogs/catalog-jdbc-mysql/build.gradle.kts b/catalogs/catalog-jdbc-mysql/build.gradle.kts index fbd2cc0aa73..d96b6209c6b 100644 --- a/catalogs/catalog-jdbc-mysql/build.gradle.kts +++ b/catalogs/catalog-jdbc-mysql/build.gradle.kts @@ -76,6 +76,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } register("copyLibAndConfig", Copy::class) { diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysql.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysql.java index 687ef1073db..417678e799c 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysql.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysql.java @@ -58,7 +58,7 @@ public static void startup() throws Exception { Collections.emptyMap()); } - private static Map getMySQLCatalogProperties() throws SQLException { + protected static Map getMySQLCatalogProperties() throws SQLException { Map catalogProperties = Maps.newHashMap(); MySQLContainer mySQLContainer = containerSuite.getMySQLContainer(); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlCatalogOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlCatalogOperations.java new file mode 100644 index 00000000000..f428c39a8eb --- /dev/null +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlCatalogOperations.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.mysql.operation; + +import java.sql.SQLException; +import org.apache.gravitino.catalog.jdbc.MySQLProtocolCompatibleCatalogOperations; +import org.apache.gravitino.catalog.mysql.MysqlCatalog; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("gravitino-docker-test") +public class TestMysqlCatalogOperations extends TestMysql { + + @Test + public void testCheckJDBCDriver() throws SQLException { + MySQLProtocolCompatibleCatalogOperations catalogOperations = + new MySQLProtocolCompatibleCatalogOperations( + null, null, DATABASE_OPERATIONS, TABLE_OPERATIONS, null); + catalogOperations.initialize(getMySQLCatalogProperties(), null, new MysqlCatalog()); + } +} diff --git a/catalogs/catalog-jdbc-postgresql/build.gradle.kts b/catalogs/catalog-jdbc-postgresql/build.gradle.kts index af89c2dbf32..f35585509c6 100644 --- a/catalogs/catalog-jdbc-postgresql/build.gradle.kts +++ b/catalogs/catalog-jdbc-postgresql/build.gradle.kts @@ -79,6 +79,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } val copyLibAndConfig by registering(Copy::class) { diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java index 04850055a16..8875ac38ba0 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java @@ -151,7 +151,8 @@ private ResultSet getSchema(Connection connection, String schemaName) throws SQL @Override public String generateDropDatabaseSql(String schema, boolean cascade) { - StringBuilder sqlBuilder = new StringBuilder(String.format("DROP SCHEMA %s", schema)); + StringBuilder sqlBuilder = + new StringBuilder(String.format("DROP SCHEMA %s%s%s", PG_QUOTE, schema, PG_QUOTE)); if (cascade) { sqlBuilder.append(" CASCADE"); } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index 9475387500a..685e04eb327 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -324,6 +324,17 @@ void testCreateUpperCaseSchemaAndTable() { Optional column = Arrays.stream(t.columns()).filter(c -> c.name().equals("binary")).findFirst(); Assertions.assertTrue(column.isPresent()); + + boolean result = tableCatalog.dropTable(tableIdentifier); + Assertions.assertTrue(result); + + Assertions.assertThrows(Exception.class, () -> tableCatalog.loadTable(tableIdentifier)); + + // Test drop schema with upper-case name + result = catalog.asSchemas().dropSchema(schemaN, false); + + // Test whether drop the schema succussfully. + Assertions.assertTrue(result); } private Map createProperties() { diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts index 8ce72eebaab..d423604949d 100644 --- a/catalogs/catalog-kafka/build.gradle.kts +++ b/catalogs/catalog-kafka/build.gradle.kts @@ -72,6 +72,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } register("copyLibAndConfig", Copy::class) { diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index f7a2514803b..8936c75c155 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -128,6 +128,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } register("copyLibAndConfig", Copy::class) { diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java index 86da0eaea62..679deb43927 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOpsHelper.java @@ -54,7 +54,6 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.types.Type.PrimitiveType; import org.apache.iceberg.types.Types.NestedField; -import org.apache.iceberg.types.Types.StructType; public class IcebergTableOpsHelper { @VisibleForTesting public static final Joiner DOT = Joiner.on("."); @@ -133,13 +132,9 @@ private void doMoveColumn( } private void doUpdateColumnPosition( - UpdateSchema icebergUpdateSchema, - UpdateColumnPosition updateColumnPosition, - Schema icebergTableSchema) { - StructType tableSchema = icebergTableSchema.asStruct(); - ColumnPosition columnPosition = - getColumnPositionForIceberg(tableSchema, updateColumnPosition.getPosition()); - doMoveColumn(icebergUpdateSchema, updateColumnPosition.fieldName(), columnPosition); + UpdateSchema icebergUpdateSchema, UpdateColumnPosition updateColumnPosition) { + doMoveColumn( + icebergUpdateSchema, updateColumnPosition.fieldName(), updateColumnPosition.getPosition()); } private void doUpdateColumnType( @@ -160,23 +155,6 @@ private void doUpdateColumnType( icebergUpdateSchema.updateColumn(fieldName, (PrimitiveType) type); } - // Iceberg doesn't support LAST position, transform to FIRST or AFTER. - private ColumnPosition getColumnPositionForIceberg( - StructType parent, ColumnPosition columnPosition) { - if (!(columnPosition instanceof TableChange.Default)) { - return columnPosition; - } - - List fields = parent.fields(); - // no column, add to first - if (fields.isEmpty()) { - return ColumnPosition.first(); - } - - NestedField last = fields.get(fields.size() - 1); - return ColumnPosition.after(last.name()); - } - private void doAddColumn(UpdateSchema icebergUpdateSchema, AddColumn addColumn) { if (addColumn.isAutoIncrement()) { throw new IllegalArgumentException("Iceberg doesn't support auto increment column"); @@ -230,8 +208,7 @@ private void alterTableColumn( } else if (change instanceof DeleteColumn) { doDeleteColumn(icebergUpdateSchema, (DeleteColumn) change, icebergTableSchema); } else if (change instanceof UpdateColumnPosition) { - doUpdateColumnPosition( - icebergUpdateSchema, (UpdateColumnPosition) change, icebergTableSchema); + doUpdateColumnPosition(icebergUpdateSchema, (UpdateColumnPosition) change); } else if (change instanceof RenameColumn) { doRenameColumn(icebergUpdateSchema, (RenameColumn) change); } else if (change instanceof UpdateColumnType) { diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index fa7577984c2..415c86d3cc7 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -589,43 +589,6 @@ void testListAndDropIcebergTable() { Assertions.assertEquals(0, tableIdentifiers.size()); } - @Test - public void testUpdateIcebergColumnDefaultPosition() { - Column col1 = Column.of("name", Types.StringType.get(), "comment"); - Column col2 = Column.of("address", Types.StringType.get(), "comment"); - Column col3 = Column.of("date_of_birth", Types.StringType.get(), "comment"); - Column[] newColumns = new Column[] {col1, col2, col3}; - NameIdentifier tableIdentifier = - NameIdentifier.of(schemaName, GravitinoITUtils.genRandomName("CatalogIcebergIT_table")); - catalog - .asTableCatalog() - .createTable( - tableIdentifier, - newColumns, - table_comment, - ImmutableMap.of(), - Transforms.EMPTY_TRANSFORM, - Distributions.NONE, - new SortOrder[0]); - - catalog - .asTableCatalog() - .alterTable( - tableIdentifier, - TableChange.updateColumnPosition( - new String[] {col1.name()}, TableChange.ColumnPosition.defaultPos())); - - Table updateColumnPositionTable = catalog.asTableCatalog().loadTable(tableIdentifier); - - Column[] updateCols = updateColumnPositionTable.columns(); - Assertions.assertEquals(3, updateCols.length); - Assertions.assertEquals(col2.name(), updateCols[0].name()); - Assertions.assertEquals(col3.name(), updateCols[1].name()); - Assertions.assertEquals(col1.name(), updateCols[2].name()); - - catalog.asTableCatalog().dropTable(tableIdentifier); - } - @Test public void testAlterIcebergTable() { Column[] columns = createColumns(); diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts index 214b5454179..4d563e28875 100644 --- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts +++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts @@ -122,6 +122,8 @@ tasks { exclude { details -> details.file.isDirectory() } + + fileMode = 0b111101101 } register("copyLibAndConfig", Copy::class) { diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/CatalogUtils.java b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/CatalogUtils.java index fcc4f3088a9..848063904f4 100644 --- a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/CatalogUtils.java +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/CatalogUtils.java @@ -24,7 +24,6 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.io.File; import java.util.Map; @@ -46,7 +45,6 @@ public class CatalogUtils { private CatalogUtils() {} - @VisibleForTesting public static PaimonBackendCatalogWrapper loadCatalogBackend(PaimonConfig paimonConfig) { Map allConfig = paimonConfig.getAllConfig(); AuthenticationConfig authenticationConfig = new AuthenticationConfig(allConfig); diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java index 4741e846255..18576e2775f 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java @@ -627,6 +627,10 @@ public void accept(ErrorResponse errorResponse) { throw new NoSuchMetalakeException(errorMessage); } else if (errorResponse.getType().equals(NoSuchRoleException.class.getSimpleName())) { throw new NoSuchRoleException(errorMessage); + } else if (errorResponse + .getType() + .equals(NoSuchMetadataObjectException.class.getSimpleName())) { + throw new NoSuchMetadataObjectException(errorMessage); } else { throw new NotFoundException(errorMessage); } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java index 962bb09e1ae..9b7769200be 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java @@ -242,11 +242,12 @@ public boolean deleteRole(String role) throws NoSuchMetalakeException { * @return The created Role instance. * @throws RoleAlreadyExistsException If a Role with the same name already exists. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws NoSuchMetadataObjectException If securable object doesn't exist * @throws RuntimeException If creating the Role encounters storage issues. */ public Role createRole( String role, Map properties, List securableObjects) - throws RoleAlreadyExistsException, NoSuchMetalakeException { + throws RoleAlreadyExistsException, NoSuchMetalakeException, NoSuchMetadataObjectException { return getMetalake().createRole(role, properties, securableObjects); } /** diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java index b7fd5d943a4..f13958cb526 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java @@ -82,6 +82,7 @@ import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.exceptions.TagAlreadyExistsException; import org.apache.gravitino.exceptions.UserAlreadyExistsException; +import org.apache.gravitino.rest.RESTUtils; import org.apache.gravitino.tag.Tag; import org.apache.gravitino.tag.TagChange; import org.apache.gravitino.tag.TagOperations; @@ -484,7 +485,7 @@ public User addUser(String user) throws UserAlreadyExistsException, NoSuchMetala public boolean removeUser(String user) throws NoSuchMetalakeException { RemoveResponse resp = restClient.delete( - String.format(API_METALAKES_USERS_PATH, this.name(), user), + String.format(API_METALAKES_USERS_PATH, this.name(), RESTUtils.encodeString(user)), RemoveResponse.class, Collections.emptyMap(), ErrorHandlers.userErrorHandler()); @@ -505,7 +506,7 @@ public boolean removeUser(String user) throws NoSuchMetalakeException { public User getUser(String user) throws NoSuchUserException, NoSuchMetalakeException { UserResponse resp = restClient.get( - String.format(API_METALAKES_USERS_PATH, this.name(), user), + String.format(API_METALAKES_USERS_PATH, this.name(), RESTUtils.encodeString(user)), UserResponse.class, Collections.emptyMap(), ErrorHandlers.userErrorHandler()); @@ -551,7 +552,7 @@ public Group addGroup(String group) throws GroupAlreadyExistsException, NoSuchMe public boolean removeGroup(String group) throws NoSuchMetalakeException { RemoveResponse resp = restClient.delete( - String.format(API_METALAKES_GROUPS_PATH, this.name(), group), + String.format(API_METALAKES_GROUPS_PATH, this.name(), RESTUtils.encodeString(group)), RemoveResponse.class, Collections.emptyMap(), ErrorHandlers.groupErrorHandler()); @@ -572,7 +573,7 @@ public boolean removeGroup(String group) throws NoSuchMetalakeException { public Group getGroup(String group) throws NoSuchGroupException, NoSuchMetalakeException { GroupResponse resp = restClient.get( - String.format(API_METALAKES_GROUPS_PATH, this.name(), group), + String.format(API_METALAKES_GROUPS_PATH, this.name(), RESTUtils.encodeString(group)), GroupResponse.class, Collections.emptyMap(), ErrorHandlers.groupErrorHandler()); @@ -593,7 +594,7 @@ public Group getGroup(String group) throws NoSuchGroupException, NoSuchMetalakeE public Role getRole(String role) throws NoSuchRoleException, NoSuchMetalakeException { RoleResponse resp = restClient.get( - String.format(API_METALAKES_ROLES_PATH, this.name(), role), + String.format(API_METALAKES_ROLES_PATH, this.name(), RESTUtils.encodeString(role)), RoleResponse.class, Collections.emptyMap(), ErrorHandlers.roleErrorHandler()); @@ -614,7 +615,7 @@ public Role getRole(String role) throws NoSuchRoleException, NoSuchMetalakeExcep public boolean deleteRole(String role) throws NoSuchMetalakeException { DeleteResponse resp = restClient.delete( - String.format(API_METALAKES_ROLES_PATH, this.name(), role), + String.format(API_METALAKES_ROLES_PATH, this.name(), RESTUtils.encodeString(role)), DeleteResponse.class, Collections.emptyMap(), ErrorHandlers.roleErrorHandler()); @@ -632,11 +633,12 @@ public boolean deleteRole(String role) throws NoSuchMetalakeException { * @return The created Role instance. * @throws RoleAlreadyExistsException If a Role with the same name already exists. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. + * @throws NoSuchMetadataObjectException If the securable object doesn't exist * @throws RuntimeException If creating the Role encounters storage issues. */ public Role createRole( String role, Map properties, List securableObjects) - throws RoleAlreadyExistsException, NoSuchMetalakeException { + throws RoleAlreadyExistsException, NoSuchMetalakeException, NoSuchMetadataObjectException { RoleCreateRequest req = new RoleCreateRequest( role, @@ -676,7 +678,10 @@ public User grantRolesToUser(List roles, String user) UserResponse resp = restClient.put( - String.format(API_PERMISSION_PATH, this.name(), String.format("users/%s/grant", user)), + String.format( + API_PERMISSION_PATH, + this.name(), + String.format("users/%s/grant", RESTUtils.encodeString(user))), request, UserResponse.class, Collections.emptyMap(), @@ -705,7 +710,9 @@ public Group grantRolesToGroup(List roles, String group) GroupResponse resp = restClient.put( String.format( - API_PERMISSION_PATH, this.name(), String.format("groups/%s/grant", group)), + API_PERMISSION_PATH, + this.name(), + String.format("groups/%s/grant", RESTUtils.encodeString(group))), request, GroupResponse.class, Collections.emptyMap(), @@ -733,7 +740,10 @@ public User revokeRolesFromUser(List roles, String user) UserResponse resp = restClient.put( - String.format(API_PERMISSION_PATH, this.name(), String.format("users/%s/revoke", user)), + String.format( + API_PERMISSION_PATH, + this.name(), + String.format("users/%s/revoke", RESTUtils.encodeString(user))), request, UserResponse.class, Collections.emptyMap(), @@ -762,7 +772,9 @@ public Group revokeRolesFromGroup(List roles, String group) GroupResponse resp = restClient.put( String.format( - API_PERMISSION_PATH, this.name(), String.format("groups/%s/revoke", group)), + API_PERMISSION_PATH, + this.name(), + String.format("groups/%s/revoke", RESTUtils.encodeString(group))), request, GroupResponse.class, Collections.emptyMap(), @@ -787,7 +799,9 @@ public Optional getOwner(MetadataObject object) throws NoSuchMetadataObje API_METALAKES_OWNERS_PATH, this.name(), String.format( - "%s/%s", object.type().name().toLowerCase(Locale.ROOT), object.fullName())), + "%s/%s", + object.type().name().toLowerCase(Locale.ROOT), + RESTUtils.encodeString(object.fullName()))), OwnerResponse.class, Collections.emptyMap(), ErrorHandlers.ownerErrorHandler()); @@ -813,7 +827,9 @@ public void setOwner(MetadataObject object, String ownerName, Owner.Type ownerTy API_METALAKES_OWNERS_PATH, this.name(), String.format( - "%s/%s", object.type().name().toLowerCase(Locale.ROOT), object.fullName())), + "%s/%s", + object.type().name().toLowerCase(Locale.ROOT), + RESTUtils.encodeString(object.fullName()))), request, SetResponse.class, Collections.emptyMap(), @@ -843,6 +859,20 @@ public GravitinoMetalake build() { } } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof GravitinoMetalake)) { + return false; + } + + GravitinoMetalake that = (GravitinoMetalake) o; + return super.equals(that); + } + /** @return the builder for creating a new instance of GravitinoMetaLake. */ public static Builder builder() { return new Builder(); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java index 48ade28d1bc..78280d2a33f 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java @@ -69,6 +69,8 @@ public class TestGravitinoMetalake extends TestBase { private static final String provider = "test"; + private static final Instant testStartTime = Instant.now(); + protected static GravitinoClient gravitinoClient; @BeforeAll @@ -701,14 +703,37 @@ public void testDeleteTag() throws JsonProcessingException { Assertions.assertTrue(ex1.getMessage().contains("mock error")); } + @Test + public void testEquals() throws JsonProcessingException { + GravitinoMetalake metalake1 = createMetalake(client, "test", true); + GravitinoMetalake metalake2 = createMetalake(client, "test", true); + GravitinoMetalake metalake3 = createMetalake(client, "another-test", true); + GravitinoMetalake metalake4 = createMetalake(client, "test"); + Assertions.assertEquals(metalake1, metalake1); + Assertions.assertEquals(metalake1, metalake2); + Assertions.assertNotEquals(metalake1, metalake3); + Assertions.assertNotEquals(metalake1, metalake4); + Assertions.assertNotEquals(metalake1, null); + Assertions.assertNotEquals(metalake1, new Object()); + } + static GravitinoMetalake createMetalake(GravitinoAdminClient client, String metalakeName) throws JsonProcessingException { + return createMetalake(client, metalakeName, false); + } + + static GravitinoMetalake createMetalake( + GravitinoAdminClient client, String metalakeName, boolean fixedAudit) + throws JsonProcessingException { MetalakeDTO mockMetalake = MetalakeDTO.builder() .withName(metalakeName) .withComment("comment") .withAudit( - AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + AuditDTO.builder() + .withCreator("creator") + .withCreateTime(fixedAudit ? testStartTime : Instant.now()) + .build()) .build(); MetalakeCreateRequest req = new MetalakeCreateRequest(metalakeName, "comment", Collections.emptyMap()); diff --git a/clients/client-python/gravitino/auth/default_oauth2_token_provider.py b/clients/client-python/gravitino/auth/default_oauth2_token_provider.py index beefc90c463..a8a98d306d8 100644 --- a/clients/client-python/gravitino/auth/default_oauth2_token_provider.py +++ b/clients/client-python/gravitino/auth/default_oauth2_token_provider.py @@ -26,7 +26,10 @@ from gravitino.dto.requests.oauth2_client_credential_request import ( OAuth2ClientCredentialRequest, ) -from gravitino.exceptions.base import GravitinoRuntimeException +from gravitino.exceptions.base import ( + GravitinoRuntimeException, + IllegalArgumentException, +) from gravitino.exceptions.handlers.oauth_error_handler import OAUTH_ERROR_HANDLER CLIENT_CREDENTIALS = "client_credentials" @@ -61,11 +64,14 @@ def __init__( self._token = self._fetch_token() def validate(self): - assert ( - self._credential and self._credential.strip() - ), "OAuth2TokenProvider must set credential" - assert self._scope and self._scope.strip(), "OAuth2TokenProvider must set scope" - assert self._path and self._path.strip(), "OAuth2TokenProvider must set path" + if not self._credential or not self._credential.strip(): + raise IllegalArgumentException("OAuth2TokenProvider must set credential") + + if not self._scope or not self._scope.strip(): + raise IllegalArgumentException("OAuth2TokenProvider must set scope") + + if not self._path or not self._path.strip(): + raise IllegalArgumentException("OAuth2TokenProvider must set path") def _get_access_token(self) -> Optional[str]: @@ -81,7 +87,8 @@ def _get_access_token(self) -> Optional[str]: return self._token def _parse_credential(self): - assert self._credential is not None, "Invalid credential: None" + if self._credential is None: + raise ValueError("Invalid credential: None") credential_info = self._credential.split(CREDENTIAL_SPLITTER, maxsplit=1) client_id = None diff --git a/clients/client-python/gravitino/catalog/base_schema_catalog.py b/clients/client-python/gravitino/catalog/base_schema_catalog.py index 2b146b27e98..70061799d0a 100644 --- a/clients/client-python/gravitino/catalog/base_schema_catalog.py +++ b/clients/client-python/gravitino/catalog/base_schema_catalog.py @@ -36,6 +36,7 @@ from gravitino.namespace import Namespace from gravitino.utils import HTTPClient from gravitino.rest.rest_utils import encode_string +from gravitino.exceptions.base import IllegalArgumentException logger = logging.getLogger(__name__) @@ -238,12 +239,13 @@ def validate(self): f"Catalog namespace must be non-null and have 1 level, the input namespace is {self._catalog_namespace}", ) - assert self.rest_client is not None, "restClient must be set" - assert ( - self.name() is not None and len(self.name().strip()) > 0 - ), "name must not be blank" - assert self.type() is not None, "type must not be None" - assert ( - self.provider() is not None and len(self.provider().strip()) > 0 - ), "provider must not be blank" - assert self.audit_info() is not None, "audit must not be None" + if self.rest_client is None: + raise IllegalArgumentException("restClient must be set") + if not self.name() or not self.name().strip(): + raise IllegalArgumentException("name must not be blank") + if self.type() is None: + raise IllegalArgumentException("type must not be None") + if not self.provider() or not self.provider().strip(): + raise IllegalArgumentException("provider must not be blank") + if self.audit_info() is None: + raise IllegalArgumentException("audit must not be None") diff --git a/clients/client-python/gravitino/client/gravitino_metalake.py b/clients/client-python/gravitino/client/gravitino_metalake.py index 9ef787a0772..1cf5c4f7295 100644 --- a/clients/client-python/gravitino/client/gravitino_metalake.py +++ b/clients/client-python/gravitino/client/gravitino_metalake.py @@ -30,6 +30,7 @@ from gravitino.dto.responses.catalog_response import CatalogResponse from gravitino.dto.responses.drop_response import DropResponse from gravitino.dto.responses.entity_list_response import EntityListResponse +from gravitino.exceptions.handlers.catalog_error_handler import CATALOG_ERROR_HANDLER from gravitino.utils import HTTPClient @@ -66,7 +67,7 @@ def list_catalogs(self) -> List[str]: A list of the catalog names under this metalake. """ url = f"api/metalakes/{self.name()}/catalogs" - response = self.rest_client.get(url) + response = self.rest_client.get(url, error_handler=CATALOG_ERROR_HANDLER) entity_list = EntityListResponse.from_json(response.body, infer_missing=True) entity_list.validate() return [identifier.name() for identifier in entity_list.identifiers()] @@ -82,7 +83,9 @@ def list_catalogs_info(self) -> List[Catalog]: """ params = {"details": "true"} url = f"api/metalakes/{self.name()}/catalogs" - response = self.rest_client.get(url, params=params) + response = self.rest_client.get( + url, params=params, error_handler=CATALOG_ERROR_HANDLER + ) catalog_list = CatalogListResponse.from_json(response.body, infer_missing=True) return [ @@ -103,7 +106,7 @@ def load_catalog(self, name: str) -> Catalog: The Catalog with specified name. """ url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) - response = self.rest_client.get(url) + response = self.rest_client.get(url, error_handler=CATALOG_ERROR_HANDLER) catalog_resp = CatalogResponse.from_json(response.body, infer_missing=True) return DTOConverters.to_catalog( @@ -145,7 +148,9 @@ def create_catalog( catalog_create_request.validate() url = f"api/metalakes/{self.name()}/catalogs" - response = self.rest_client.post(url, json=catalog_create_request) + response = self.rest_client.post( + url, json=catalog_create_request, error_handler=CATALOG_ERROR_HANDLER + ) catalog_resp = CatalogResponse.from_json(response.body, infer_missing=True) return DTOConverters.to_catalog( @@ -172,7 +177,9 @@ def alter_catalog(self, name: str, *changes: CatalogChange) -> Catalog: updates_request.validate() url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) - response = self.rest_client.put(url, json=updates_request) + response = self.rest_client.put( + url, json=updates_request, error_handler=CATALOG_ERROR_HANDLER + ) catalog_response = CatalogResponse.from_json(response.body, infer_missing=True) catalog_response.validate() @@ -191,7 +198,7 @@ def drop_catalog(self, name: str) -> bool: """ try: url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) - response = self.rest_client.delete(url) + response = self.rest_client.delete(url, error_handler=CATALOG_ERROR_HANDLER) drop_response = DropResponse.from_json(response.body, infer_missing=True) drop_response.validate() diff --git a/clients/client-python/gravitino/client/gravitino_version.py b/clients/client-python/gravitino/client/gravitino_version.py index c69376f1b18..ae42b990c6c 100644 --- a/clients/client-python/gravitino/client/gravitino_version.py +++ b/clients/client-python/gravitino/client/gravitino_version.py @@ -22,7 +22,7 @@ from enum import Enum from gravitino.dto.version_dto import VersionDTO -from gravitino.exceptions.base import GravitinoRuntimeException +from gravitino.exceptions.base import BadRequestException, GravitinoRuntimeException class Version(Enum): @@ -51,7 +51,8 @@ def __init__(self, versionDTO): m = re.match(VERSION_PATTERN, self.version()) - assert m is not None, "Invalid version string " + self.version() + if m is None: + raise BadRequestException(f"Invalid version string: {self.version()}") self.major = int(m.group(Version.MAJOR.value)) self.minor = int(m.group(Version.MINOR.value)) diff --git a/clients/client-python/gravitino/constants/error.py b/clients/client-python/gravitino/constants/error.py index e4b0ab61e5f..877d523ad3a 100644 --- a/clients/client-python/gravitino/constants/error.py +++ b/clients/client-python/gravitino/constants/error.py @@ -20,6 +20,7 @@ from enum import IntEnum from gravitino.exceptions.base import ( + ConnectionFailedException, RESTException, IllegalArgumentException, NotFoundException, @@ -54,6 +55,9 @@ class ErrorConstants(IntEnum): # Error codes for unsupported operation. UNSUPPORTED_OPERATION_CODE = 1006 + # Error codes for connect to catalog failed. + CONNECTION_FAILED_CODE = 1007 + # Error codes for invalid state. UNKNOWN_ERROR_CODE = 1100 @@ -66,6 +70,7 @@ class ErrorConstants(IntEnum): AlreadyExistsException: ErrorConstants.ALREADY_EXISTS_CODE, NotEmptyException: ErrorConstants.NON_EMPTY_CODE, UnsupportedOperationException: ErrorConstants.UNSUPPORTED_OPERATION_CODE, + ConnectionFailedException: ErrorConstants.CONNECTION_FAILED_CODE, } ERROR_CODE_MAPPING = {v: k for k, v in EXCEPTION_MAPPING.items()} diff --git a/clients/client-python/gravitino/dto/dto_converters.py b/clients/client-python/gravitino/dto/dto_converters.py index 86e614557a6..a55f053b35f 100644 --- a/clients/client-python/gravitino/dto/dto_converters.py +++ b/clients/client-python/gravitino/dto/dto_converters.py @@ -83,6 +83,8 @@ def to_catalog_update_request(change: CatalogChange): change.property(), change.value() ) if isinstance(change, CatalogChange.RemoveProperty): - return CatalogUpdateRequest.RemoveCatalogPropertyRequest(change.property()) + return CatalogUpdateRequest.RemoveCatalogPropertyRequest( + change.get_property() + ) raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/dto/requests/catalog_create_request.py b/clients/client-python/gravitino/dto/requests/catalog_create_request.py index c8f70dd11df..853cb5cf5b8 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_create_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_create_request.py @@ -58,10 +58,9 @@ def validate(self): Raises: IllegalArgumentException if name or type are not set. """ - assert self._name is not None, '"name" field is required and cannot be empty' - assert ( - self._type is not None - ), '"catalog_type" field is required and cannot be empty' - assert ( - self._provider is not None - ), '"provider" field is required and cannot be empty' + if not self._name: + raise ValueError('"name" field is required and cannot be empty') + if not self._type: + raise ValueError('"catalog_type" field is required and cannot be empty') + if not self._provider: + raise ValueError('"provider" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/dto/requests/catalog_update_request.py b/clients/client-python/gravitino/dto/requests/catalog_update_request.py index d3ba6298afc..6ae06b118f9 100644 --- a/clients/client-python/gravitino/dto/requests/catalog_update_request.py +++ b/clients/client-python/gravitino/dto/requests/catalog_update_request.py @@ -62,9 +62,8 @@ def validate(self): Raises: IllegalArgumentException if the new name is not set. """ - assert ( - self._new_name is None - ), '"newName" field is required and cannot be empty' + if not self._new_name: + raise ValueError('"newName" field is required and cannot be empty') @dataclass class UpdateCatalogCommentRequest(CatalogUpdateRequestBase): @@ -81,9 +80,8 @@ def catalog_change(self): return CatalogChange.update_comment(self._new_comment) def validate(self): - assert ( - self._new_comment is None - ), '"newComment" field is required and cannot be empty' + if not self._new_comment: + raise ValueError('"newComment" field is required and cannot be empty') @dataclass class SetCatalogPropertyRequest(CatalogUpdateRequestBase): @@ -104,10 +102,10 @@ def catalog_change(self): return CatalogChange.set_property(self._property, self._value) def validate(self): - assert ( - self._property is None - ), '"property" field is required and cannot be empty' - assert self._value is None, '"value" field is required and cannot be empty' + if not self._property: + raise ValueError('"property" field is required and cannot be empty') + if not self._value: + raise ValueError('"value" field is required and cannot be empty') class RemoveCatalogPropertyRequest(CatalogUpdateRequestBase): """Request to remove a property from a catalog.""" @@ -123,6 +121,5 @@ def catalog_change(self): return CatalogChange.remove_property(self._property) def validate(self): - assert ( - self._property is None - ), '"property" field is required and cannot be empty' + if not self._property: + raise ValueError('"property" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/dto/requests/schema_create_request.py b/clients/client-python/gravitino/dto/requests/schema_create_request.py index 5e09e2f2bc1..e5df0a8bd3b 100644 --- a/clients/client-python/gravitino/dto/requests/schema_create_request.py +++ b/clients/client-python/gravitino/dto/requests/schema_create_request.py @@ -43,4 +43,5 @@ def __init__( self._properties = properties def validate(self): - assert self._name is not None, '"name" field is required and cannot be empty' + if not self._name: + raise ValueError('"name" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/dto/responses/base_response.py b/clients/client-python/gravitino/dto/responses/base_response.py index 8ce91bd9b0f..52cdfe9fbc1 100644 --- a/clients/client-python/gravitino/dto/responses/base_response.py +++ b/clients/client-python/gravitino/dto/responses/base_response.py @@ -20,6 +20,7 @@ from dataclasses import dataclass, field from dataclasses_json import config +from gravitino.exceptions.base import IllegalArgumentException from gravitino.rest.rest_message import RESTResponse @@ -37,4 +38,5 @@ def validate(self): """Validates the response code. @throws IllegalArgumentException if code value is negative. """ - assert self._code >= 0, "code must be >= 0" + if self._code < 0: + raise IllegalArgumentException("code must be >= 0") diff --git a/clients/client-python/gravitino/dto/responses/catalog_response.py b/clients/client-python/gravitino/dto/responses/catalog_response.py index 7118165638d..f1e68798e2c 100644 --- a/clients/client-python/gravitino/dto/responses/catalog_response.py +++ b/clients/client-python/gravitino/dto/responses/catalog_response.py @@ -20,6 +20,7 @@ from dataclasses import dataclass, field from dataclasses_json import config +from gravitino.exceptions.base import IllegalArgumentException from .base_response import BaseResponse from ..catalog_dto import CatalogDTO @@ -39,12 +40,14 @@ def validate(self): """ super().validate() - assert self.catalog is not None, "catalog must not be null" - assert ( - self.catalog.name() is not None - ), "catalog 'name' must not be null and empty" - assert self.catalog.type() is not None, "catalog 'type' must not be null" - assert self.catalog.audit_info() is not None, "catalog 'audit' must not be null" + if self._catalog is None: + raise IllegalArgumentException("catalog must not be null") + if self._catalog.name() is None: + raise IllegalArgumentException("catalog 'name' must not be null and empty") + if self._catalog.type() is None: + raise IllegalArgumentException("catalog 'type' must not be null") + if self._catalog.audit_info() is None: + raise IllegalArgumentException("catalog 'audit' must not be null") def catalog(self) -> CatalogDTO: return self._catalog diff --git a/clients/client-python/gravitino/dto/responses/entity_list_response.py b/clients/client-python/gravitino/dto/responses/entity_list_response.py index 4e32ac4e48e..522eb4edce7 100644 --- a/clients/client-python/gravitino/dto/responses/entity_list_response.py +++ b/clients/client-python/gravitino/dto/responses/entity_list_response.py @@ -23,6 +23,7 @@ from dataclasses_json import config from gravitino.dto.responses.base_response import BaseResponse +from gravitino.exceptions.base import IllegalArgumentException from gravitino.name_identifier import NameIdentifier @@ -43,4 +44,5 @@ def validate(self): """ super().validate() - assert self._idents is not None, "identifiers must not be null" + if self._idents is None: + raise IllegalArgumentException("identifiers must not be null") diff --git a/clients/client-python/gravitino/dto/responses/error_response.py b/clients/client-python/gravitino/dto/responses/error_response.py index ad1fd273e94..1c4a0703c55 100644 --- a/clients/client-python/gravitino/dto/responses/error_response.py +++ b/clients/client-python/gravitino/dto/responses/error_response.py @@ -23,7 +23,7 @@ from gravitino.dto.responses.base_response import BaseResponse from gravitino.constants.error import ErrorConstants, EXCEPTION_MAPPING -from gravitino.exceptions.base import UnknownError +from gravitino.exceptions.base import BadRequestException, UnknownError @dataclass @@ -46,12 +46,11 @@ def stack(self): def validate(self): super().validate() - assert ( - self._type is not None and len(self._type) != 0 - ), "type cannot be None or empty" - assert ( - self._message is not None and len(self._message) != 0 - ), "message cannot be None or empty" + if self._type is None or not self._type.strip(): + raise BadRequestException("Type cannot be None or empty") + + if self._message is None or not self._message.strip(): + raise BadRequestException("Message cannot be None or empty") def __repr__(self) -> str: return ( diff --git a/clients/client-python/gravitino/dto/responses/fileset_response.py b/clients/client-python/gravitino/dto/responses/fileset_response.py index db0a1d88609..f6460f433b3 100644 --- a/clients/client-python/gravitino/dto/responses/fileset_response.py +++ b/clients/client-python/gravitino/dto/responses/fileset_response.py @@ -23,6 +23,7 @@ from gravitino.dto.fileset_dto import FilesetDTO from gravitino.dto.responses.base_response import BaseResponse +from gravitino.exceptions.base import IllegalArgumentException @dataclass @@ -41,11 +42,13 @@ def validate(self): IllegalArgumentException if catalog identifiers are not set. """ super().validate() - assert self._fileset is not None, "fileset must not be null" - assert self._fileset.name, "fileset 'name' must not be null and empty" - assert ( - self._fileset.storage_location - ), "fileset 'storageLocation' must not be null and empty" - assert ( - self._fileset.type is not None - ), "fileset 'type' must not be null and empty" + if self._fileset is None: + raise IllegalArgumentException("fileset must not be null") + if not self._fileset.name(): + raise IllegalArgumentException("fileset 'name' must not be null and empty") + if not self._fileset.storage_location(): + raise IllegalArgumentException( + "fileset 'storageLocation' must not be null and empty" + ) + if self._fileset.type() is None: + raise IllegalArgumentException("fileset 'type' must not be null and empty") diff --git a/clients/client-python/gravitino/dto/responses/metalake_response.py b/clients/client-python/gravitino/dto/responses/metalake_response.py index c3c4be1033b..996e98ed915 100644 --- a/clients/client-python/gravitino/dto/responses/metalake_response.py +++ b/clients/client-python/gravitino/dto/responses/metalake_response.py @@ -24,6 +24,7 @@ from gravitino.dto.metalake_dto import MetalakeDTO from gravitino.dto.responses.base_response import BaseResponse +from gravitino.exceptions.base import IllegalArgumentException @dataclass @@ -41,10 +42,11 @@ def validate(self): """ super().validate() - assert self._metalake is not None, "metalake must not be null" - assert ( - self._metalake.name() is not None - ), "metalake 'name' must not be null and empty" - assert ( - self._metalake.audit_info() is not None - ), "metalake 'audit' must not be null" + if self._metalake is None: + raise IllegalArgumentException("Metalake must not be null") + + if self._metalake.name() is None: + raise IllegalArgumentException("Metalake 'name' must not be null and empty") + + if self._metalake.audit_info() is None: + raise IllegalArgumentException("Metalake 'audit' must not be null") diff --git a/clients/client-python/gravitino/dto/responses/oauth2_error_response.py b/clients/client-python/gravitino/dto/responses/oauth2_error_response.py index f7e472c13f3..20edf171d44 100644 --- a/clients/client-python/gravitino/dto/responses/oauth2_error_response.py +++ b/clients/client-python/gravitino/dto/responses/oauth2_error_response.py @@ -21,6 +21,7 @@ from dataclasses_json import config from gravitino.dto.responses.error_response import ErrorResponse +from gravitino.exceptions.base import IllegalArgumentException @dataclass @@ -37,4 +38,5 @@ def message(self): return self._message def validate(self): - assert self._type is not None, "OAuthErrorResponse should contain type" + if self._type is None: + raise IllegalArgumentException("OAuthErrorResponse should contain type") diff --git a/clients/client-python/gravitino/dto/responses/oauth2_token_response.py b/clients/client-python/gravitino/dto/responses/oauth2_token_response.py index 07869ec034c..37071b72372 100644 --- a/clients/client-python/gravitino/dto/responses/oauth2_token_response.py +++ b/clients/client-python/gravitino/dto/responses/oauth2_token_response.py @@ -23,6 +23,7 @@ from gravitino.dto.responses.base_response import BaseResponse from gravitino.auth.auth_constants import AuthConstants +from gravitino.exceptions.base import IllegalArgumentException @dataclass @@ -45,11 +46,16 @@ def validate(self): """ super().validate() - assert self._access_token is not None, "Invalid access token: None" - assert ( + if self._access_token is None: + raise IllegalArgumentException("Invalid access token: None") + + if ( AuthConstants.AUTHORIZATION_BEARER_HEADER.strip().lower() - == self._token_type.lower() - ), f'Unsupported token type: {self._token_type} (must be "bearer")' + != self._token_type.lower() + ): + raise IllegalArgumentException( + f'Unsupported token type: {self._token_type} (must be "bearer")' + ) def access_token(self) -> str: return self._access_token diff --git a/clients/client-python/gravitino/dto/responses/schema_response.py b/clients/client-python/gravitino/dto/responses/schema_response.py index 836b5376e80..ed6f04caf36 100644 --- a/clients/client-python/gravitino/dto/responses/schema_response.py +++ b/clients/client-python/gravitino/dto/responses/schema_response.py @@ -23,6 +23,7 @@ from gravitino.dto.responses.base_response import BaseResponse from gravitino.dto.schema_dto import SchemaDTO +from gravitino.exceptions.base import IllegalArgumentException @dataclass @@ -44,8 +45,11 @@ def validate(self): """ super().validate() - assert self._schema is not None, "schema must be non-null" - assert ( - self._schema.name() is not None - ), "schema 'name' must not be null and empty" - assert self._schema.audit_info() is not None, "schema 'audit' must not be null" + if self._schema is None: + raise IllegalArgumentException("Schema must be non-null") + + if self._schema.name() is None: + raise IllegalArgumentException("Schema 'name' must not be null and empty") + + if self._schema.audit_info() is None: + raise IllegalArgumentException("Schema 'audit' must not be null") diff --git a/clients/client-python/gravitino/dto/responses/version_response.py b/clients/client-python/gravitino/dto/responses/version_response.py index abeebcabf12..acb6ca808b2 100644 --- a/clients/client-python/gravitino/dto/responses/version_response.py +++ b/clients/client-python/gravitino/dto/responses/version_response.py @@ -22,6 +22,7 @@ from gravitino.dto.responses.base_response import BaseResponse from gravitino.dto.version_dto import VersionDTO +from gravitino.exceptions.base import IllegalArgumentException @dataclass @@ -41,13 +42,20 @@ def validate(self): """ super().validate() - assert self._version is not None, "version must be non-null" - assert ( - self._version.version() is not None - ), "version 'version' must not be null and empty" - assert ( - self._version.compile_date() is not None - ), "version 'compile_date' must not be null and empty" - assert ( - self._version.git_commit() is not None - ), "version 'git_commit' must not be null and empty" + if self._version is None: + raise IllegalArgumentException("Version must be non-null") + + if self._version.version() is None: + raise IllegalArgumentException( + "Version 'version' must not be null or empty" + ) + + if self._version.compile_date() is None: + raise IllegalArgumentException( + "Version 'compile_date' must not be null or empty" + ) + + if self._version.git_commit() is None: + raise IllegalArgumentException( + "Version 'git_commit' must not be null or empty" + ) diff --git a/clients/client-python/gravitino/exceptions/base.py b/clients/client-python/gravitino/exceptions/base.py index a7d6eee1831..674121a5d0a 100644 --- a/clients/client-python/gravitino/exceptions/base.py +++ b/clients/client-python/gravitino/exceptions/base.py @@ -83,6 +83,10 @@ class SchemaAlreadyExistsException(AlreadyExistsException): """An exception thrown when a schema already exists.""" +class CatalogAlreadyExistsException(AlreadyExistsException): + """An exception thrown when a resource already exists.""" + + class NotEmptyException(GravitinoRuntimeException): """Base class for all exceptions thrown when a resource is not empty.""" @@ -95,6 +99,10 @@ class UnknownError(RuntimeError): """An exception thrown when other unknown exception is thrown""" +class ConnectionFailedException(GravitinoRuntimeException): + """An exception thrown when connect to catalog failed.""" + + class UnauthorizedException(GravitinoRuntimeException): """An exception thrown when a user is not authorized to perform an action.""" diff --git a/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py b/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py new file mode 100644 index 00000000000..e714edad39e --- /dev/null +++ b/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py @@ -0,0 +1,52 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from gravitino.constants.error import ErrorConstants +from gravitino.dto.responses.error_response import ErrorResponse +from gravitino.exceptions.handlers.rest_error_handler import RestErrorHandler +from gravitino.exceptions.base import ( + ConnectionFailedException, + NoSuchMetalakeException, + NoSuchCatalogException, + CatalogAlreadyExistsException, +) + + +class CatalogErrorHandler(RestErrorHandler): + + def handle(self, error_response: ErrorResponse): + + error_message = error_response.format_error_message() + code = error_response.code() + exception_type = error_response.type() + + if code == ErrorConstants.CONNECTION_FAILED_CODE: + raise ConnectionFailedException(error_message) + if code == ErrorConstants.NOT_FOUND_CODE: + if exception_type == NoSuchMetalakeException.__name__: + raise NoSuchMetalakeException(error_message) + if exception_type == NoSuchCatalogException.__name__: + raise NoSuchCatalogException(error_message) + if code == ErrorConstants.ALREADY_EXISTS_CODE: + raise CatalogAlreadyExistsException(error_message) + + super().handle(error_response) + + +CATALOG_ERROR_HANDLER = CatalogErrorHandler() diff --git a/clients/client-python/gravitino/namespace.py b/clients/client-python/gravitino/namespace.py index 221fd3ea8d4..0fa23922e41 100644 --- a/clients/client-python/gravitino/namespace.py +++ b/clients/client-python/gravitino/namespace.py @@ -40,9 +40,10 @@ def to_json(self): @classmethod def from_json(cls, levels): - assert levels is not None and isinstance( - levels, list - ), f"Cannot parse name identifier from invalid JSON: {levels}" + if levels is None or not isinstance(levels, list): + raise IllegalNamespaceException( + f"Cannot parse name identifier from invalid JSON: {levels}" + ) return cls(levels) @staticmethod diff --git a/clients/client-python/gravitino/rest/rest_utils.py b/clients/client-python/gravitino/rest/rest_utils.py index 077511a2f52..7e31633ec60 100644 --- a/clients/client-python/gravitino/rest/rest_utils.py +++ b/clients/client-python/gravitino/rest/rest_utils.py @@ -18,10 +18,12 @@ """ import urllib.parse +from gravitino.exceptions.base import IllegalArgumentException def encode_string(to_encode: str): - assert to_encode is not None, "Invalid string to encode: None" + if to_encode is None: + raise IllegalArgumentException("Invalid string to encode: None") return urllib.parse.quote(to_encode) diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index bf5447dd1c0..1b404162fbf 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -32,15 +32,25 @@ version="0.7.0.dev0", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/apache/gravitino", - author="apache-gravitino", + author="Apache Software Foundation", author_email="dev@gravitino.apache.org", + maintainer="Apache Gravitino Community", + maintainer_email="dev@gravitino.apache.org", + license="Apache-2.0", + url="https://github.com/apache/gravitino", python_requires=">=3.8", + keywords="Data, AI, metadata, catalog", packages=find_packages(exclude=["tests*", "scripts*"]), + project_urls={ + "Homepage": "https://gravitino.apache.org/", + "Source Code": "https://github.com/apache/gravitino", + "Documentation": "https://gravitino.apache.org/docs/overview", + "Bug Tracker": "https://github.com/apache/gravitino/issues", + "Slack Chat": "https://the-asf.slack.com/archives/C078RESTT19", + }, classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/clients/client-python/tests/integration/hdfs_container.py b/clients/client-python/tests/integration/hdfs_container.py index b9e1887fc8e..176fb052c9d 100644 --- a/clients/client-python/tests/integration/hdfs_container.py +++ b/clients/client-python/tests/integration/hdfs_container.py @@ -27,6 +27,7 @@ from docker.errors import NotFound from gravitino.exceptions.base import GravitinoRuntimeException +from gravitino.exceptions.base import InternalError logger = logging.getLogger(__name__) @@ -63,7 +64,8 @@ async def check_hdfs_container_status(hdfs_container): result = await asyncio.wait_for( check_hdfs_status(hdfs_container), timeout=timeout_sec ) - assert result is True, "HDFS container startup failed!" + if not result: + raise InternalError("HDFS container startup failed!") except asyncio.TimeoutError as e: raise GravitinoRuntimeException( "Timeout occurred while waiting for checking HDFS container status." diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 3f8d2ae4b53..2cf13f2cdcf 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -26,6 +26,11 @@ GravitinoClient, Catalog, ) +from gravitino.api.catalog_change import CatalogChange +from gravitino.exceptions.base import ( + CatalogAlreadyExistsException, + NoSuchCatalogException, +) from tests.integration.integration_test_env import IntegrationTestEnv @@ -36,6 +41,7 @@ class TestCatalog(IntegrationTestEnv): metalake_name: str = "TestSchema_metalake" + str(randint(1, 10000)) catalog_name: str = "testCatalog" + catalog_comment: str = "catalogComment" catalog_location_prop: str = "location" # Fileset Catalog must set `location` catalog_provider: str = "hadoop" @@ -59,11 +65,13 @@ def init_test_env(self): self.gravitino_client = GravitinoClient( uri="http://localhost:8090", metalake_name=self.metalake_name ) - self.gravitino_client.create_catalog( - name=self.catalog_name, + + def create_catalog(self, catalog_name) -> Catalog: + return self.gravitino_client.create_catalog( + name=catalog_name, catalog_type=Catalog.Type.FILESET, provider=self.catalog_provider, - comment="", + comment=self.catalog_comment, properties={self.catalog_location_prop: "/tmp/test_schema"}, ) @@ -86,5 +94,67 @@ def clean_test_data(self): logger.error("Clean test data failed: %s", e) def test_list_catalogs(self): + self.create_catalog(self.catalog_name) catalog_names = self.gravitino_client.list_catalogs() self.assertTrue(self.catalog_name in catalog_names) + + def test_create_catalog(self): + catalog = self.create_catalog(self.catalog_name) + self.assertEqual(catalog.name(), self.catalog_name) + self.assertEqual( + catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} + ) + + def test_failed_create_catalog(self): + self.create_catalog(self.catalog_name) + with self.assertRaises(CatalogAlreadyExistsException): + _ = self.create_catalog(self.catalog_name) + + def test_alter_catalog(self): + catalog = self.create_catalog(self.catalog_name) + + catalog_new_name = self.catalog_name + "_new" + catalog_new_comment = self.catalog_comment + "_new" + catalog_properties_new_value = self.catalog_location_prop + "_new" + catalog_properties_new_key: str = "catalog_properties_new_key" + + changes = ( + CatalogChange.rename(catalog_new_name), + CatalogChange.update_comment(catalog_new_comment), + CatalogChange.set_property( + catalog_properties_new_key, catalog_properties_new_value + ), + ) + + catalog = self.gravitino_client.alter_catalog(self.catalog_name, *changes) + self.assertEqual(catalog.name(), catalog_new_name) + self.assertEqual(catalog.comment(), catalog_new_comment) + self.assertEqual( + catalog.properties().get(catalog_properties_new_key), + catalog_properties_new_value, + ) + self.catalog_name = self.catalog_name + "_new" + + def test_drop_catalog(self): + self.create_catalog(self.catalog_name) + self.assertTrue(self.gravitino_client.drop_catalog(name=self.catalog_name)) + + def test_list_catalogs_info(self): + self.create_catalog(self.catalog_name) + catalogs_info = self.gravitino_client.list_catalogs_info() + self.assertTrue(any(item.name() == self.catalog_name for item in catalogs_info)) + + def test_load_catalog(self): + self.create_catalog(self.catalog_name) + catalog = self.gravitino_client.load_catalog(self.catalog_name) + self.assertIsNotNone(catalog) + self.assertEqual(catalog.name(), self.catalog_name) + self.assertEqual(catalog.comment(), self.catalog_comment) + self.assertEqual( + catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} + ) + self.assertEqual(catalog.audit_info().creator(), "anonymous") + + def test_failed_load_catalog(self): + with self.assertRaises(NoSuchCatalogException): + self.gravitino_client.load_catalog(self.catalog_name) diff --git a/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py b/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py index 7d9ef9e255f..447d165701b 100644 --- a/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py +++ b/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py @@ -22,7 +22,11 @@ from gravitino.auth.auth_constants import AuthConstants from gravitino.auth.default_oauth2_token_provider import DefaultOAuth2TokenProvider -from gravitino.exceptions.base import BadRequestException, UnauthorizedException +from gravitino.exceptions.base import ( + BadRequestException, + IllegalArgumentException, + UnauthorizedException, +) from tests.unittests.auth import mock_base OAUTH_PORT = 1082 @@ -32,13 +36,13 @@ class TestOAuth2TokenProvider(unittest.TestCase): def test_provider_init_exception(self): - with self.assertRaises(AssertionError): + with self.assertRaises(IllegalArgumentException): _ = DefaultOAuth2TokenProvider(uri="test") - with self.assertRaises(AssertionError): + with self.assertRaises(IllegalArgumentException): _ = DefaultOAuth2TokenProvider(uri="test", credential="xx") - with self.assertRaises(AssertionError): + with self.assertRaises(IllegalArgumentException): _ = DefaultOAuth2TokenProvider(uri="test", credential="xx", scope="test") @patch( @@ -75,7 +79,7 @@ def test_authertication_invalid_grant_error(self, *mock_methods): ) def test_authentication_with_error_authentication_type(self, *mock_methods): - with self.assertRaises(AssertionError): + with self.assertRaises(IllegalArgumentException): _ = DefaultOAuth2TokenProvider( uri=f"http://127.0.0.1:{OAUTH_PORT}", credential="yy:xx", diff --git a/clients/client-python/tests/unittests/test_error_handler.py b/clients/client-python/tests/unittests/test_error_handler.py index fd92af495ac..b2695471f54 100644 --- a/clients/client-python/tests/unittests/test_error_handler.py +++ b/clients/client-python/tests/unittests/test_error_handler.py @@ -34,11 +34,14 @@ NotEmptyException, SchemaAlreadyExistsException, UnsupportedOperationException, + ConnectionFailedException, + CatalogAlreadyExistsException, ) from gravitino.exceptions.handlers.rest_error_handler import REST_ERROR_HANDLER from gravitino.exceptions.handlers.fileset_error_handler import FILESET_ERROR_HANDLER from gravitino.exceptions.handlers.metalake_error_handler import METALAKE_ERROR_HANDLER +from gravitino.exceptions.handlers.catalog_error_handler import CATALOG_ERROR_HANDLER from gravitino.exceptions.handlers.schema_error_handler import SCHEMA_ERROR_HANDLER @@ -151,6 +154,46 @@ def test_metalake_error_handler(self): ErrorResponse.generate_error_response(Exception, "mock error") ) + def test_catalog_error_handler(self): + + with self.assertRaises(ConnectionFailedException): + CATALOG_ERROR_HANDLER.handle( + ErrorResponse.generate_error_response( + ConnectionFailedException, "mock error" + ) + ) + + with self.assertRaises(NoSuchMetalakeException): + CATALOG_ERROR_HANDLER.handle( + ErrorResponse.generate_error_response( + NoSuchMetalakeException, "mock error" + ) + ) + + with self.assertRaises(NoSuchCatalogException): + CATALOG_ERROR_HANDLER.handle( + ErrorResponse.generate_error_response( + NoSuchCatalogException, "mock error" + ) + ) + + with self.assertRaises(CatalogAlreadyExistsException): + CATALOG_ERROR_HANDLER.handle( + ErrorResponse.generate_error_response( + CatalogAlreadyExistsException, "mock error" + ) + ) + + with self.assertRaises(InternalError): + CATALOG_ERROR_HANDLER.handle( + ErrorResponse.generate_error_response(InternalError, "mock error") + ) + + with self.assertRaises(RESTException): + CATALOG_ERROR_HANDLER.handle( + ErrorResponse.generate_error_response(Exception, "mock error") + ) + def test_schema_error_handler(self): with self.assertRaises(NoSuchCatalogException): diff --git a/clients/client-python/tests/unittests/test_gravitino_version.py b/clients/client-python/tests/unittests/test_gravitino_version.py index 6f18bcfd162..37a4a7b6895 100644 --- a/clients/client-python/tests/unittests/test_gravitino_version.py +++ b/clients/client-python/tests/unittests/test_gravitino_version.py @@ -21,7 +21,7 @@ from gravitino.client.gravitino_version import GravitinoVersion from gravitino.dto.version_dto import VersionDTO -from gravitino.exceptions.base import GravitinoRuntimeException +from gravitino.exceptions.base import BadRequestException, GravitinoRuntimeException class TestGravitinoVersion(unittest.TestCase): @@ -57,16 +57,12 @@ def test_parse_version_string(self): self.assertEqual(version.patch, 0) # Test an invalid the version string with 2 part - self.assertRaises( - AssertionError, GravitinoVersion, VersionDTO("0.6", "2023-01-01", "1234567") - ) + with self.assertRaises(BadRequestException): + GravitinoVersion(VersionDTO("0.6", "2023-01-01", "1234567")) # Test an invalid the version string with not number - self.assertRaises( - AssertionError, - GravitinoVersion, - VersionDTO("a.b.c", "2023-01-01", "1234567"), - ) + with self.assertRaises(BadRequestException): + GravitinoVersion(VersionDTO("a.b.c", "2023-01-01", "1234567")) def test_version_compare(self): # test equal diff --git a/common/src/main/java/org/apache/gravitino/dto/authorization/SecurableObjectDTO.java b/common/src/main/java/org/apache/gravitino/dto/authorization/SecurableObjectDTO.java index 8a10d93df1a..71014799406 100644 --- a/common/src/main/java/org/apache/gravitino/dto/authorization/SecurableObjectDTO.java +++ b/common/src/main/java/org/apache/gravitino/dto/authorization/SecurableObjectDTO.java @@ -31,9 +31,6 @@ /** Data transfer object representing a securable object. */ public class SecurableObjectDTO implements SecurableObject { - @JsonProperty("fullName") - private String fullName; - @JsonProperty("type") private Type type; @@ -46,27 +43,27 @@ public class SecurableObjectDTO implements SecurableObject { /** Default constructor for Jackson deserialization. */ protected SecurableObjectDTO() {} + /** @return The full name of the securable object. */ + @JsonProperty("fullName") + public String getFullName() { + return fullName(); + } + /** - * Creates a new instance of SecurableObject DTO. + * Sets the full name of the securable object. Only used by Jackson deserializer. * - * @param privileges The privileges of the SecurableObject DTO. - * @param fullName The name of the SecurableObject DTO. - * @param type The type of the securable object. + * @param fullName The full name of the metadata object. */ - protected SecurableObjectDTO(String fullName, Type type, PrivilegeDTO[] privileges) { - this.type = type; - this.fullName = fullName; + @JsonProperty("fullName") + public void setFullName(String fullName) { int index = fullName.lastIndexOf("."); - if (index == -1) { - this.parent = null; - this.name = fullName; + parent = null; + name = fullName; } else { - this.parent = fullName.substring(0, index); - this.name = fullName.substring(index + 1); + parent = fullName.substring(0, index); + name = fullName.substring(index + 1); } - - this.privileges = privileges; } @Nullable @@ -80,11 +77,6 @@ public String name() { return name; } - @Override - public String fullName() { - return fullName; - } - @Override public Type type() { return type; @@ -106,10 +98,13 @@ public static Builder builder() { /** Builder for {@link SecurableObjectDTO}. */ public static class Builder { + private final SecurableObjectDTO securableObjectDTO = new SecurableObjectDTO(); private String fullName; private Type type; private PrivilegeDTO[] privileges; + private Builder() {} + /** * Sets the full name of the securable object. * @@ -158,9 +153,11 @@ public SecurableObjectDTO build() { Preconditions.checkArgument( privileges != null && privileges.length != 0, "privileges can't be null or empty"); - SecurableObjectDTO object = new SecurableObjectDTO(fullName, type, privileges); + securableObjectDTO.type = type; + securableObjectDTO.privileges = privileges; + securableObjectDTO.setFullName(fullName); - return object; + return securableObjectDTO; } } } diff --git a/conf/gravitino-env.sh.template b/conf/gravitino-env.sh.template index 656ca176b2b..1d39e2e8ff6 100644 --- a/conf/gravitino-env.sh.template +++ b/conf/gravitino-env.sh.template @@ -17,6 +17,10 @@ # under the License. # +# In build.gradle.kts, we would automatic replace GRAVITINO_VERSION to the current version of Gravitino +# when running `./gradlew compileDistribution` or `./gradlew compileIcebergRESTServer` command. +GRAVITINO_VERSION=GRAVITINO_VERSION_PLACEHOLDER + # Debug Gravitino server # export GRAVITINO_DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000 -Dlog4j2.debug=true" diff --git a/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java b/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java index fd6efc70f5f..5317b19a4dd 100644 --- a/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java +++ b/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java @@ -59,4 +59,7 @@ private ConfigConstants() {} /** The version number for the 0.6.0 release. */ public static final String VERSION_0_6_0 = "0.6.0"; + + /** The version number for the 0.7.0 release. */ + public static final String VERSION_0_7_0 = "0.7.0"; } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java index 7b52d34a147..93bf0b5f668 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -162,7 +162,7 @@ TagPO selectTagMetaByMetalakeAndName( + " WHERE tag_id = #{oldTagMeta.tagId}" + " AND metalake_id = #{oldTagMeta.metalakeId}" + " AND tag_name = #{oldTagMeta.tagName}" - + " AND tag_comment = #{oldTagMeta.comment}" + + " AND (tag_comment IS NULL OR tag_comment = #{oldTagMeta.comment})" + " AND properties = #{oldTagMeta.properties}" + " AND audit_info = #{oldTagMeta.auditInfo}" + " AND current_version = #{oldTagMeta.currentVersion}" diff --git a/dev/ci/util_free_space.sh b/dev/ci/util_free_space.sh index c7ab15b6402..51fb3459b70 100755 --- a/dev/ci/util_free_space.sh +++ b/dev/ci/util_free_space.sh @@ -49,6 +49,10 @@ if [ "${GITHUB_ACTIONS}" = "true" ]; then sudo rm -rf /opt/hostedtoolcache/PyPy || : # 376MB sudo rm -rf /opt/hostedtoolcache/node || : + # Free at least 10G, android is useless for Gravitino CI. + sudo rm -rf /usr/local/lib/android || : + # Free at lease 5G, .ghcup is installed by Haskell CI and useless for Gravitino CI. + sudo rm -rf /usr/local/.ghcup || : # Remove Web browser packages if dpkg-query -l firefox;then sudo apt purge -y firefox diff --git a/dev/docker/iceberg-rest-server/Dockerfile b/dev/docker/iceberg-rest-server/Dockerfile index a85f6707275..d4a85915ff0 100644 --- a/dev/docker/iceberg-rest-server/Dockerfile +++ b/dev/docker/iceberg-rest-server/Dockerfile @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -FROM openjdk:17 +FROM openjdk:17-jdk-buster LABEL maintainer="dev@gravitino.apache.org" diff --git a/dev/release/update-java-doc-version.sh b/dev/release/update-java-doc-version.sh new file mode 100755 index 00000000000..13a29c37d20 --- /dev/null +++ b/dev/release/update-java-doc-version.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +NEW_VERSION=$1 +cd "$(cd "$(dirname "$0")" && pwd)/../../docs" +CURRENT_VERSION=`cat index.md| grep pathname:///docs | head -n 1 | awk -F '///docs' '{print $2}' | awk -F '/' '{print $2}'` + +if [[ "${NEW_VERSION}" == "${CURRENT_VERSION}" ]]; then + echo "The new version is the same as the current version." + exit 1 +fi + +# Detect the operating system +if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + find "$(pwd)" -name "*.md" | xargs sed -i '' "s|/docs/${CURRENT_VERSION}/api|/docs/${NEW_VERSION}/api|g" +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + # Linux + find "$(pwd)" -name "*.md" | xargs sed -i "s|/docs/${CURRENT_VERSION}/api|/docs/${NEW_VERSION}/api|g" +else + echo "Unsupported OS" + exit 1 +fi + +echo "Update the version from ${CURRENT_VERSION} to ${NEW_VERSION} successfully." \ No newline at end of file diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md index d0287a26fa9..063108d9dcb 100644 --- a/docs/apache-hive-catalog.md +++ b/docs/apache-hive-catalog.md @@ -16,7 +16,8 @@ Apache Gravitino offers the capability to utilize [Apache Hive](https://hive.apa * Gravitino must have network access to the Hive metastore service using the Thrift protocol. :::note -The Hive catalog is available for Apache Hive **2.x** only. Support for Apache Hive 3.x is under development. +Although the Hive catalog uses the Hive2 metastore client, it can be compatible with the Hive3 metastore service because the called HMS APIs are still available in Hive3. +If there is any compatibility issue, please create an [issue](https://github.com/apache/gravitino/issues). ::: ## Catalog diff --git a/docs/assets/object.png b/docs/assets/object.png new file mode 100644 index 00000000000..9eafcc2fec9 Binary files /dev/null and b/docs/assets/object.png differ diff --git a/docs/assets/privilege.png b/docs/assets/privilege.png new file mode 100644 index 00000000000..cef89e2715b Binary files /dev/null and b/docs/assets/privilege.png differ diff --git a/docs/assets/role.png b/docs/assets/role.png new file mode 100644 index 00000000000..4345dd8f23c Binary files /dev/null and b/docs/assets/role.png differ diff --git a/docs/assets/trino/create-gravitino-connector.jpg b/docs/assets/trino/create-gravitino-trino-connector.jpg similarity index 100% rename from docs/assets/trino/create-gravitino-connector.jpg rename to docs/assets/trino/create-gravitino-trino-connector.jpg diff --git a/docs/assets/user-group.png b/docs/assets/user-group.png new file mode 100644 index 00000000000..b26773e3e3a Binary files /dev/null and b/docs/assets/user-group.png differ diff --git a/docs/assets/workflow.png b/docs/assets/workflow.png new file mode 100644 index 00000000000..83a4bae24ca Binary files /dev/null and b/docs/assets/workflow.png differ diff --git a/docs/flink-connector/flink-catalog-hive.md b/docs/flink-connector/flink-catalog-hive.md index 1ca84c105cc..136dac3ed20 100644 --- a/docs/flink-connector/flink-catalog-hive.md +++ b/docs/flink-connector/flink-catalog-hive.md @@ -1,6 +1,6 @@ --- title: "Flink connector hive catalog" -slug: /flinkk-connector/flink-catalog-hive +slug: /flink-connector/flink-catalog-hive keyword: flink connector hive catalog license: "This software is licensed under the Apache License version 2." --- diff --git a/docs/flink-connector/flink-connector.md b/docs/flink-connector/flink-connector.md index c14186a2d07..948e4554b15 100644 --- a/docs/flink-connector/flink-connector.md +++ b/docs/flink-connector/flink-connector.md @@ -50,7 +50,7 @@ TableEnvironment tableEnv = TableEnvironment.create(builder.inBatchMode().build( 3. Execute the Flink SQL query. -Suppose there is only one hive catalog with the name hive in the metalake test. +Suppose there is only one hive catalog with the name `hive` in the metalake `test`. ```sql // use hive catalog diff --git a/docs/getting-started.md b/docs/getting-started.md index 01220ae675b..420d583d78a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -317,7 +317,7 @@ After completing these steps, you should be able to access the Gravitino REST in 1. **Explore documentation:** - Delve deeper into the Gravitino documentation for advanced features and configuration options. - - Check out https://datastrato.ai/docs + - Check out https://gravitino.apache.org/docs/latest 2. **Community engagement:** - Join the Gravitino community forums to connect with other users, share experiences, and seek assistance if needed. @@ -325,12 +325,12 @@ After completing these steps, you should be able to access the Gravitino REST in - Check out our Slack channel in ASF Slack: https://the-asf.slack.com 3. **Read our blogs:** - - Check out: https://gravitino.apache.org/blog (coming soon) + - Check out: https://gravitino.apache.org/blog 4. **Continuous updates:** - Stay informed about Gravitino updates and new releases to benefit from the latest features, optimizations, and security enhancements. - - Check out our Website: https://gravitino.apache.org (coming soon) + - Check out our Website: https://gravitino.apache.org This document is just the beginning. You're welcome to customize your Gravitino setup based on your requirements and to explore the vast possibilities this powerful tool offers. If you encounter any issues or have questions, you can always connect with the Gravitino community for assistance. diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 4f3fba6c751..785269024a0 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -140,7 +140,7 @@ For more details, please refer to the definition of the plugin. ### Security configuration -Refer to [security](security.md) for HTTPS and authentication configurations. +Refer to [security](security/security.md) for HTTPS and authentication configurations. ### Metrics configuration diff --git a/docs/how-to-build.md b/docs/how-to-build.md index 3bd51427071..a46389e7256 100644 --- a/docs/how-to-build.md +++ b/docs/how-to-build.md @@ -267,4 +267,4 @@ Access [http://localhost:8090](http://localhost:8090) Building the Gravitino project compiles the necessary components, and starting the server allows you to access the application in your browser. -For instructions on how to run the project using VSCode or IntelliJ on Windows, please refer to [CONTRIBUTING.md](CONTRIBUTING.md). +For instructions on how to run the project using VSCode or IntelliJ on Windows, please refer to [CONTRIBUTING.md](https://github.com/apache/gravitino/blob/main/CONTRIBUTING.md). diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md index 96c7d449500..ec3dd6ccd38 100644 --- a/docs/how-to-use-gvfs.md +++ b/docs/how-to-use-gvfs.md @@ -274,7 +274,7 @@ FileSystem fs = filesetPath.getFileSystem(conf); ##### Using `OAuth` authentication If you want to use `oauth2` authentication for the Gravitino client in the Gravitino Virtual File System, -please refer to this document to complete the configuration of the Gravitino server and the OAuth server: [Security](./security.md). +please refer to this document to complete the configuration of the Gravitino server and the OAuth server: [Security](security/security.md). Then, you can configure the Hadoop configuration like this: @@ -298,7 +298,7 @@ FileSystem fs = filesetPath.getFileSystem(conf); ##### Using `Kerberos` authentication If you want to use `kerberos` authentication for the Gravitino client in the Gravitino Virtual File System, -please refer to this document to complete the configuration of the Gravitino server: [Security](./security.md). +please refer to this document to complete the configuration of the Gravitino server: [Security](security/security.md). Then, you can configure the Hadoop configuration like this: @@ -362,7 +362,7 @@ fs = gvfs.GravitinoVirtualFileSystem(server_uri="http://localhost:8090", metalak You can get it by [pip](https://pip.pypa.io/en/stable/installation/): ```shell - pip install gravitino + pip install apache-gravitino ``` 2. Configuring the Hadoop environment. diff --git a/docs/how-to-use-python-client.md b/docs/how-to-use-python-client.md index 76698ff4dbb..5b3f4aa0c4b 100644 --- a/docs/how-to-use-python-client.md +++ b/docs/how-to-use-python-client.md @@ -5,7 +5,7 @@ date: 2024-05-09 keyword: Gravitino Python client license: This software is licensed under the Apache License version 2. --- -## Introduction +# Apache Gravitino Python client Apache Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the metadata directly in different sources, types, and regions, also provides users @@ -26,7 +26,7 @@ install it in your local. ### Apache Gravitino Python client API ```shell -pip install gravitino +pip install apache-gravitino ``` 1. [Manage metalake using Gravitino Python API](./manage-metalake-using-gravitino.md?language=python) @@ -36,7 +36,7 @@ pip install gravitino We offer a playground environment to help you quickly understand how to use Gravitino Python client to manage non-tabular data on HDFS via Fileset in Gravitino. You can refer to the -document [How to use the playground#Launch AI components of playground](./how-to-use-the-playground.md#launch-ai-components-of-playground) +document [How to use the playground](./how-to-use-the-playground.md) to launch a Gravitino server, HDFS and Jupyter notebook environment in you local Docker environment. Waiting for the playground Docker environment to start, you can directly open @@ -111,7 +111,7 @@ You can ues any IDE to develop Gravitino Python Client. Directly open the client ./gradlew :clients:client-python:distribution ``` -6. Deploy the Gravitino Python client to https://pypi.org/project/gravitino/ +6. Deploy the Gravitino Python client to https://pypi.org/project/apache-gravitino/ ```shell ./gradlew :clients:client-python:deploy @@ -125,3 +125,15 @@ You can ues any IDE to develop Gravitino Python Client. Directly open the client + User documentation: https://datastrato.ai/docs/ + Videos on Youtube: https://www.youtube.com/@Datastrato + Slack Community: [https://the-asf.slack.com#gravitino](https://the-asf.slack.com/archives/C078RESTT19) + +## License + +Gravitino is under the Apache License Version 2.0, See the [LICENSE](https://github.com/apache/gravitino/blob/main/LICENSE) for the details. + +## ASF Incubator disclaimer + +Apache Gravitino is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. +Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, +and decision making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability of the code, +it does indicate that the project has yet to be fully endorsed by the ASF. diff --git a/docs/how-to-use-relational-backend-storage.md b/docs/how-to-use-relational-backend-storage.md index 0ccb57f6b87..cc282d9d736 100644 --- a/docs/how-to-use-relational-backend-storage.md +++ b/docs/how-to-use-relational-backend-storage.md @@ -6,9 +6,8 @@ license: "This software is licensed under the Apache License version 2." ## Introduction -Before the version `0.5.0`, Apache Gravitino only supports KV backend storage to store metadata. Since -RDBMS is widely used in the industry, starting from the version `0.5.0`, Gravitino supports using -RDBMS as relational backend storage to store metadata. This doc will guide you on how to use the +Before the version `0.6.0`, Apache Gravitino supports KV and Relational backend storage to store metadata. +Since 0.6.0, Gravitino only supports using RDBMS as relational backend storage to store metadata. This doc will guide you on how to use the relational backend storage in Gravitino. Relational backend storage mainly aims to the users who are accustomed to using RDBMS to @@ -17,12 +16,12 @@ store data or lack available a KV storage, and want to use Gravitino. With relational backend storage, you can quickly deploy Gravitino in a production environment and take advantage of relational storage to manage metadata. -### What kind of backend storage are supported +### What kind of backend storage is supported -Currently, relational backend storage supports the `JDBCBackend`, and uses `MySQL` as the +Currently, relational backend storage supports the `JDBCBackend`, and `MySQL` and `H2` are supported currently for `JDBCBackend`, `H2` is the default storage for `JDBCBackend`. -## How to use +## How to use MySQL ### Prerequisites @@ -87,3 +86,7 @@ Finally, you can run the script in the distribution package directory to start t ```shell ./${GRAVITINO_HOME}/bin/gravitino.sh start ``` + +## How to use H2 + +As mentioned above, `H2` is the default storage for `JDBCBackend`, so you can use `H2` directly without any additional configuration. \ No newline at end of file diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 6ae17f74bf6..c1a4a8111d6 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -60,7 +60,7 @@ Starting with version `0.6.0`, the prefix `gravitino.auxService.iceberg-rest.` f Please note that, it only takes affect in `gravitino.conf`, you don't need to specify the above configurations if start as a standalone server. -### REST catalog server configuration +### HTTP server configuration | Configuration item | Description | Default value | Required | Since Version | |--------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|----------|---------------| @@ -81,7 +81,22 @@ You can also specify filter parameters by setting configuration entries in the s ### Security -Gravitino Iceberg REST server supports OAuth2 and HTTPS, please refer to [Security](./security.md) for more details. +Gravitino Iceberg REST server supports OAuth2 and HTTPS, please refer to [Security](security/security.md) for more details. + +#### Backend authentication + +For JDBC backend, you can use the `gravitino.iceberg-rest.jdbc.user` and `gravitino.iceberg-rest.jdbc.password` to authenticate the JDBC connection. For Hive backend, you can use the `gravitino.iceberg-rest.authentication.type` to specify the authentication type, and use the `gravitino.iceberg-rest.authentication.kerberos.principal` and `gravitino.iceberg-rest.authentication.kerberos.keytab-uri` to authenticate the Kerberos connection. +The detailed configuration items are as follows: + +| Configuration item | Description | Default value | Required | Since Version | +|---------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------|---------------| +| `gravitino.iceberg-rest.authentication.type` | The type of authentication for Iceberg rest catalog backend. This configuration only applicable for for Hive backend, and only supports `Kerberos`, `simple` currently. As for JDBC backend, only username/password authentication was supported now. | `simple` | No | 0.6.0 | +| `gravitino.iceberg-rest.authentication.impersonation-enable` | Whether to enable impersonation for the Iceberg catalog | `false` | No | 0.6.0 | +| `gravitino.iceberg-rest.authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. | 0.6.0 | +| `gravitino.iceberg-rest.authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `gravitino.iceberg-rest.authentication.type` is Kerberos. | 0.6.0 | +| `gravitino.iceberg-rest.authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Iceberg catalog. | 60 | No | 0.6.0 | +| `gravitino.iceberg-rest.authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.6.0 | + ### Storage @@ -113,33 +128,33 @@ You should place HDFS configuration file to the classpath of the Iceberg REST se Builds with Hadoop 2.10.x. There may be compatibility issues when accessing Hadoop 3.x clusters. ::: -### Apache Gravitino Iceberg catalog backend configuration +### Catalog backend configuration :::info The Gravitino Iceberg REST catalog service uses the memory catalog backend by default. You can specify a Hive or JDBC catalog backend for production environment. ::: -#### Apache Hive backend configuration +#### Hive backend configuration -| Configuration item | Description | Default value | Required | Since Version | -|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------|----------|---------------| -| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`hive`** for a Hive catalog. | `memory` | Yes | 0.2.0 | -| `gravitino.iceberg-rest.uri` | The Hive metadata address, such as `thrift://127.0.0.1:9083`. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.warehouse` | The warehouse directory of the Hive catalog, such as `/user/hive/warehouse-hive/`. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.catalog-backend-name` | The catalog backend name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `hive` for Hive backend, `jdbc` for JDBC backend, `memory` for memory backend | No | 0.5.2 | +| Configuration item | Description | Default value | Required | Since Version | +|---------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|----------|---------------| +| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`hive`** for a Hive catalog. | `memory` | Yes | 0.2.0 | +| `gravitino.iceberg-rest.uri` | The Hive metadata address, such as `thrift://127.0.0.1:9083`. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.warehouse` | The warehouse directory of the Hive catalog, such as `/user/hive/warehouse-hive/`. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.catalog-backend-name` | The catalog backend name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `hive` for Hive backend, `jdbc` for JDBC backend, `memory` for memory backend | No | 0.5.2 | #### JDBC backend configuration -| Configuration item | Description | Default value | Required | Since Version | -|-----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------| -| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for a JDBC catalog. | `memory` | Yes | 0.2.0 | -| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.warehouse ` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | -| `gravitino.iceberg-rest.jdbc.user` | The username of the JDBC connection. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.jdbc.password` | The password of the JDBC connection. | (none) | Yes | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | -| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | +| Configuration item | Description | Default value | Required | Since Version | +|---------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|--------------------------|----------|---------------| +| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for a JDBC catalog. | `memory` | Yes | 0.2.0 | +| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.warehouse ` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 | +| `gravitino.iceberg-rest.jdbc.user` | The username of the JDBC connection. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.jdbc.password` | The password of the JDBC connection. | (none) | Yes | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 | +| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 | If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to keep consistent with your Jdbc Iceberg catalog name to operate the prior namespace and tables. @@ -147,6 +162,51 @@ If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to You must download the corresponding JDBC driver to the `iceberg-rest-server/libs` directory. ::: +#### Multi catalog support + +The Gravitino Iceberg REST server supports multiple catalogs and offers a configuration-based catalog management system. + +| Configuration item | Description | Default value | Required | Since Version | +|----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------|----------|---------------| +| `gravitino.iceberg-rest.catalog-provider` | Catalog provider class name, you can develop a class that implements `IcebergTableOpsProvider` and add the corresponding jar file to the Iceberg REST service classpath directory. | `config-based-provider` | No | 0.7.0 | + +When using a config-based catalog provider, you can configure the default catalog with `gravitino.iceberg-rest.catalog.=`. For specific catalogs, use the format `gravitino.iceberg-rest.catalog..=`. + +For instance, you can configure three different catalogs, the default catalog and the specific `hive_backend` and `jdbc_backend` catalogs separately. + +```text +gravitino.iceberg-rest.catalog-backend = jdbc +gravitino.iceberg-rest.uri = jdbc:postgresql://127.0.0.1:5432 +gravitino.iceberg-rest.warehouse = hdfs://127.0.0.1:9000/user/hive/warehouse-postgresql +... +gravitino.iceberg-rest.catalog.hive_backend.catalog-backend = hive +gravitino.iceberg-rest.catalog.hive_backend.uri = thrift://127.0.0.1:9084 +gravitino.iceberg-rest.catalog.hive_backend.warehouse = /user/hive/warehouse-hive/ +... +gravitino.iceberg-rest.catalog.jdbc_backend.catalog-backend = jdbc +gravitino.iceberg-rest.catalog.jdbc_backend.uri = jdbc:mysql://127.0.0.1:3306/ +gravitino.iceberg-rest.catalog.jdbc_backend.warehouse = hdfs://127.0.0.1:9000/user/hive/warehouse-mysql +... +``` + +You can access different catalogs by setting the `prefix` to the specific catalog name in the Iceberg REST client configuration. The default catalog will be used if you do not specify a `prefix`. For instance, consider the case of SparkSQL. + +```shell +./bin/spark-sql -v \ +... +--conf spark.sql.catalog.default_rest_catalog.type=rest \ +--conf spark.sql.catalog.default_rest_catalog.uri=http://127.0.0.1:9001/iceberg/ \ +... +--conf spark.sql.catalog.hive_backend_catalog.type=rest \ +--conf spark.sql.catalog.hive_backend_catalog.uri=http://127.0.0.1:9001/iceberg/ \ +--conf spark.sql.catalog.hive_backend_catalog.prefix=hive_backend \ +... +--conf spark.sql.catalog.jdbc_backend_catalog.type=rest \ +--conf spark.sql.catalog.jdbc_backend_catalog.uri=http://127.0.0.1:9001/iceberg/ \ +--conf spark.sql.catalog.jdbc_backend_catalog.prefix=jdbc_backend \ +... +``` + ### Other Apache Iceberg catalog properties You can add other properties defined in [Iceberg catalog properties](https://iceberg.apache.org/docs/1.5.2/configuration/#catalog-properties). @@ -162,7 +222,7 @@ The `clients` property for example: ### Apache Iceberg metrics store configuration -Gravitino provides a pluggable metrics store interface to store and delete Iceberg metrics. You can develop a class that implements `org.apache.gravitino.catalog.lakehouse.iceberg.web.metrics` and add the corresponding jar file to the Iceberg REST service classpath directory. +Gravitino provides a pluggable metrics store interface to store and delete Iceberg metrics. You can develop a class that implements `org.apache.gravitino.iceberg.service.metrics.IcebergMetricsStore` and add the corresponding jar file to the Iceberg REST service classpath directory. | Configuration item | Description | Default value | Required | Since Version | |-------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------| @@ -232,7 +292,7 @@ SELECT * FROM dml.test; ## Docker instructions -You could run Gravitino server though docker container: +You could run Gravitino Iceberg REST server though docker container: ```shell docker run -d -p 9001:9001 datastrato/iceberg-rest-server:0.6 diff --git a/docs/index.md b/docs/index.md index fb73ca18f19..0d680236fcb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,7 +14,7 @@ metadata access for data and AI assets. ## Downloading -You can get Gravitino from the [GitHub release page](https://github.com/apache/gravitino/releases), +You can get Gravitino from the [download page](https://gravitino.apache.org/downloads), or you can build Gravitino from source code. See [How to build Gravitino](./how-to-build.md). Gravitino runs on both Linux and macOS platforms, and it requires the installation of Java 8, Java 11, or Java 17. Gravitino trino-connector runs with @@ -58,8 +58,9 @@ REST API and the Java SDK. You can use either to manage metadata. See messaging metadata. Also, you can find the complete REST API definition in -[Gravitino Open API](./api/rest/gravitino-rest-api), and the -Java SDK definition in [Gravitino Javadoc](pathname:///docs/0.5.1/api/java/index.html). +[Gravitino Open API](./api/rest/gravitino-rest-api), +Java SDK definition in [Gravitino Java doc](pathname:///docs/0.6.0-incubating/api/java/index.html), +and Python SDK definition in [Gravitino Python doc](pathname:///docs/0.6.0-incubating/api/python/gravitino.html). Gravitino provides a web UI to manage the metadata. Visit the web UI in the browser via `http://:8090`. See [Gravitino web UI](./webui.md) for details. @@ -82,9 +83,6 @@ Gravitino currently supports the following catalogs: * [**Kafka catalog**](./kafka-catalog.md) -Gravitino also provides an Iceberg REST catalog service for the Iceberg table format. See the -[Iceberg REST catalog service](./iceberg-rest-service.md) for details. - ## Apache Gravitino playground To experience Gravitino with other components easily, Gravitino provides a playground to run. It @@ -115,6 +113,13 @@ Gravitino supports different catalogs to manage the metadata in different source using Hadoop Compatible File System (HCFS). * [Kafka catalog](./kafka-catalog.md): a complete guide to using Gravitino to manage Kafka topics metadata. +### Governance + +Gravitino provides governance features to manage metadata in a unified way. See: + +* [Manage tags in Gravitino](./manage-tags-in-gravitino.md): a complete guide to using Gravitino + to manage tags. + ### Gravitino Iceberg REST catalog service * [Iceberg REST catalog service](./iceberg-rest-service.md): a complete guide to using Gravitino as an Apache Iceberg REST catalog service. @@ -144,16 +149,23 @@ Gravitino provides a Flink connector to manage metadata in a unified way. To use Gravitino provides several ways to configure and manage the Gravitino server. See: -* [Security](./security.md): provides security configurations for Gravitino, including HTTPS - and OAuth2 configurations. * [Gravitino metrics](./metrics.md): provides metrics configurations and detailed a metrics list of the Gravitino server. +### Security + +Gravitino provides security configurations for Gravitino, including HTTPS, authentication and access control configurations. + +* [HTTPS](./security/how-to-use-https.md): provides HTTPS configurations. +* [Authentication](./security/how-to-authenticate.md): provides authentication configurations including simple, OAuth, Kerberos. +* [Access Control](./security/access-control.md): provides access control configurations. +* [CORS](./security/how-to-use-cors.md): provides CORS configurations. + ### Programming guides -* [Gravitino Open API](./api/rest/gravitino-rest-api): provides the complete Open API definition of - Gravitino. -* [Gravitino Javadoc](pathname:///docs/0.5.1/api/java/index.html): provides the Javadoc for the Gravitino API. +* [Gravitino Open API](./api/rest/gravitino-rest-api): provides the complete Open API definition of Gravitino. +* [Gravitino Java doc](pathname:///docs/0.6.0-incubating/api/java/index.html): provides the Javadoc for the Gravitino API. +* [Gravitino Python doc](pathname:///docs/0.6.0-incubating/api/python/gravitino.html) provides the Pydoc for the Gravitino API. ### Development guides diff --git a/docs/jdbc-doris-catalog.md b/docs/jdbc-doris-catalog.md index 1548455afd6..560f0baaead 100644 --- a/docs/jdbc-doris-catalog.md +++ b/docs/jdbc-doris-catalog.md @@ -148,6 +148,19 @@ Unsupported for now. +### Table partitions + +The Doris catalog supports partitioned tables. +Users can create partitioned tables in the Doris catalog with specific partitioning attributes. It is also supported to pre-assign partitions when creating Doris tables. +Note that although Gravitino supports several partitioning strategies, Apache Doris inherently only supports these two partitioning strategies: + +- `RANGE` +- `LIST` + +:::caution +The `fieldName` specified in the partitioning attributes must be the name of columns defined in the table. +::: + ### Table operations Please refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metadata-using-gravitino.md#table-operations) for more details. diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 1b330d30fe8..f20487fa155 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -25,7 +25,7 @@ Builds with Apache Iceberg `1.5.2`. The Apache Iceberg table format version is ` ### Catalog capabilities -- Works as a catalog proxy, supporting `HiveCatalog`, `JdbcCatalog` and `RESTCatalog`. +- Works as a catalog proxy, supporting `Hive`, `JDBC` and `REST` as catalog backend. - Supports DDL operations for Iceberg schemas and tables. - Doesn't support snapshot or table management operations. - Supports S3 and HDFS storage. @@ -33,30 +33,24 @@ Builds with Apache Iceberg `1.5.2`. The Apache Iceberg table format version is ` ### Catalog properties -| Property name | Description | Default value | Required | Since Version | -|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|-------------------------------------------------------------|---------------| -| `catalog-backend` | Catalog backend of Gravitino Iceberg catalog. Supports `hive` or `jdbc` or `rest`. | (none) | Yes | 0.2.0 | -| `uri` | The URI configuration of the Iceberg catalog. `thrift://127.0.0.1:9083` or `jdbc:postgresql://127.0.0.1:5432/db_name` or `jdbc:mysql://127.0.0.1:3306/metastore_db` or `http://127.0.0.1:9001`. | (none) | Yes | 0.2.0 | -| `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-hive/` for local fs or `hdfs://namespace/hdfs/path` for HDFS. | (none) | Yes | 0.2.0 | -| `catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | Gravitino catalog name | No | 0.5.2 | -| `authentication.type` | The type of authentication for Iceberg catalog backend, currently Gravitino only supports `Kerberos`, `simple`. | `simple` | No | 0.6.0 | -| `authentication.impersonation-enable` | Whether to enable impersonation for the Iceberg catalog | `false` | No | 0.6.0 | -| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.6.0 | -| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.6.0 | -| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Iceberg catalog. | 60 | No | 0.6.0 | -| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.6.0 | +| Property name | Description | Default value | Required | Since Version | +|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|-------------------------------------------------------------|---------------| +| `catalog-backend` | Catalog backend of Gravitino Iceberg catalog. Supports `hive` or `jdbc` or `rest`. | (none) | Yes | 0.2.0 | +| `uri` | The URI configuration of the Iceberg catalog. `thrift://127.0.0.1:9083` or `jdbc:postgresql://127.0.0.1:5432/db_name` or `jdbc:mysql://127.0.0.1:3306/metastore_db` or `http://127.0.0.1:9001`. | (none) | Yes | 0.2.0 | +| `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-hive/` for local fs or `hdfs://namespace/hdfs/path` for HDFS. | (none) | Yes | 0.2.0 | +| `catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | Gravitino catalog name | No | 0.5.2 | -Any properties not defined by Gravitino with `gravitino.bypass.` prefix will pass to Iceberg catalog properties and HDFS configuration. For example, if specify `gravitino.bypass.list-all-tables`, `list-all-tables` will pass to Iceberg catalog properties. +Any property not defined by Gravitino with `gravitino.bypass.` prefix will pass to Iceberg catalog properties and HDFS configuration. For example, if specify `gravitino.bypass.list-all-tables`, `list-all-tables` will pass to Iceberg catalog properties. -When you use the Gravitino with Trino. You can pass the Trino Iceberg connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.iceberg.table-statistics-enabled` to pass the `iceberg.table-statistics-enabled` to the Gravitino Iceberg catalog in Trino runtime. +If you are using the Gravitino with Trino, you can pass the Trino Iceberg connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.iceberg.table-statistics-enabled` to pass the `iceberg.table-statistics-enabled` to the Gravitino Iceberg catalog in Trino runtime. -When you use the Gravitino with Spark. You can pass the Spark Iceberg connector configuration using prefix `spark.bypass.`. For example, using `spark.bypass.io-impl` to pass the `io-impl` to the Spark Iceberg connector in Spark runtime. +If you are using the Gravitino with Spark, you can pass the Spark Iceberg connector configuration using prefix `spark.bypass.`. For example, using `spark.bypass.io-impl` to pass the `io-impl` to the Spark Iceberg connector in Spark runtime. -#### JDBC catalog +#### JDBC backend -If you are using JDBC catalog, you must provide `jdbc-user`, `jdbc-password` and `jdbc-driver` to catalog properties. +If you are using JDBC backend, you must provide properties like `jdbc-user`, `jdbc-password` and `jdbc-driver`. | Property name | Description | Default value | Required | Since Version | |-------------------|---------------------------------------------------------------------------------------------------------|---------------|----------|---------------| @@ -68,7 +62,7 @@ If you are using JDBC catalog, you must provide `jdbc-user`, `jdbc-password` and If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to keep consistent with your Jdbc Iceberg catalog name to operate the prior namespace and tables. :::caution -You must download the corresponding JDBC driver to the `catalogs/lakehouse-iceberg/libs` directory. +You must download the corresponding JDBC driver and place it to the `catalogs/lakehouse-iceberg/libs` directory If you are using JDBC backend. ::: #### S3 @@ -89,6 +83,19 @@ For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`, you Please set `gravitino.iceberg-rest.warehouse` to `s3://{bucket_name}/${prefix_name}` for JDBC catalog backend, `s3a://{bucket_name}/${prefix_name}` for Hive catalog backend. ::: +#### Catalog backend security + +Users can use the following properties to configure the security of the catalog backend if needed. For example, if you are using a Kerberos Hive catalog backend, you must set `authentication.type` to `Kerberos` and provide `authentication.kerberos.principal` and `authentication.kerberos.keytab-uri`. + +| Property name | Description | Default value | Required | Since Version | +|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|-------------------------------------------------------------|---------------| +| `authentication.type` | The type of authentication for Iceberg catalog backend. This configuration only applicable for for Hive backend, and only supports `Kerberos`, `simple` currently. As for JDBC backend, only username/password authentication was supported now. | `simple` | No | 0.6.0 | +| `authentication.impersonation-enable` | Whether to enable impersonation for the Iceberg catalog | `false` | No | 0.6.0 | +| `authentication.kerberos.principal` | The principal of the Kerberos authentication | (none) | required if the value of `authentication.type` is Kerberos. | 0.6.0 | +| `authentication.kerberos.keytab-uri` | The URI of The keytab for the Kerberos authentication. | (none) | required if the value of `authentication.type` is Kerberos. | 0.6.0 | +| `authentication.kerberos.check-interval-sec` | The check interval of Kerberos credential for Iceberg catalog. | 60 | No | 0.6.0 | +| `authentication.kerberos.keytab-fetch-timeout-sec` | The fetch timeout of retrieving Kerberos keytab from `authentication.kerberos.keytab-uri`. | 60 | No | 0.6.0 | + ### Catalog operations Please refer to [Manage Relational Metadata Using Gravitino](./manage-relational-metadata-using-gravitino.md#catalog-operations) for more details. diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md index 7265073a524..2b669ff0be6 100644 --- a/docs/lakehouse-paimon-catalog.md +++ b/docs/lakehouse-paimon-catalog.md @@ -1,6 +1,6 @@ --- title: "Paimon catalog" -slug: /lakehouse-Paimon-catalog +slug: /lakehouse-paimon-catalog keywords: - lakehouse - Paimon @@ -26,7 +26,6 @@ Builds with Apache Paimon `0.8.0`. - Supports DDL operations for Paimon schemas and tables. - Doesn't support `JdbcCatalog` and `HiveCatalog` catalog backend now. -- Doesn't support alterTable now. - Doesn't support alterSchema. ### Catalog properties @@ -72,17 +71,36 @@ Please refer to [Manage Relational Metadata Using Gravitino](./manage-relational ### Table capabilities -- Supporting createTable, dropTable, loadTable and listTable. +- Supporting createTable, dropTable, alterTable, loadTable and listTable. ``` dropTable will delete the table location directly, similar with purgeTable. ``` -- Supporting Column default value through table properties, such as `fields.{columnName}.default-value`. +- Supporting Column default value through table properties, such as `fields.{columnName}.default-value`, not column expression. + +- Doesn't support table distribution and sort orders. -- Doesn't support alterTable now. +:::info +Paimon does not support auto increment column. +::: + +#### Table changes + +- AddColumn +- DeleteColumn +- RenameColumn +- UpdateColumnComment +- UpdateColumnNullability +- UpdateColumnPosition +- UpdateColumnType +- UpdateComment +- SetProperty +- RemoveProperty #### Table partitions -- Doesn't support table partition now. +- Only supports Identity partitions, such as `day`, `hour`, etc. + +Please refer to [Paimon DDL Create Table](https://paimon.apache.org/docs/0.8/spark/sql-ddl/#create-table) for more details. ### Table sort orders @@ -90,11 +108,19 @@ dropTable will delete the table location directly, similar with purgeTable. ### Table distributions -- Only supporting `NoneDistribution` now. +- Doesn't support table distributions. ### Table indexes -- Doesn't support table indexes. +- Only supports primary key Index. + +:::info +We cannot specify more than one primary key Index, and a primary key Index can contain multiple fields as a joint primary key. +::: + +:::info +Paimon Table primary key constraint should not be same with partition fields, this will result in only one record in a partition. +::: ### Table column types @@ -131,10 +157,16 @@ You can pass [Paimon table properties](https://paimon.apache.org/docs/0.8/mainte The Gravitino server doesn't allow passing the following reserved fields. -| Configuration item | Description | -|---------------------------------|----------------------------------------------------------| -| `comment` | The table comment. | -| `creator` | The table creator. | +| Configuration item | Description | +|------------------------------------|--------------------------------------------------------------| +| `comment` | The table comment. | +| `owner` | The table owner. | +| `bucket-key` | The table bucket-key. | +| `merge-engine` | The table merge-engine. | +| `sequence.field` | The table sequence.field. | +| `rowkind.field` | The table rowkind.field. | +| `primary-key` | The table primary-key. | +| `partition` | The table partition. | ### Table operations diff --git a/docs/manage-relational-metadata-using-gravitino.md b/docs/manage-relational-metadata-using-gravitino.md index 8e06fa142db..228be9f7a53 100644 --- a/docs/manage-relational-metadata-using-gravitino.md +++ b/docs/manage-relational-metadata-using-gravitino.md @@ -21,8 +21,9 @@ For more details, please refer to the related doc. - [**Apache Hive**](./apache-hive-catalog.md) - [**MySQL**](./jdbc-mysql-catalog.md) - [**PostgreSQL**](./jdbc-postgresql-catalog.md) -- [**Doris**](./jdbc-doris-catalog.md) +- [**Apache Doris**](./jdbc-doris-catalog.md) - [**Apache Iceberg**](./lakehouse-iceberg-catalog.md) +- [**Apache Paimon**](./lakehouse-paimon-catalog.md) Assuming: @@ -76,7 +77,7 @@ Map hiveProperties = ImmutableMap.builder() Catalog catalog = gravitinoClient.createCatalog("catalog", Type.RELATIONAL, - "hive", // provider, We support hive, jdbc-mysql, jdbc-postgresql, lakehouse-iceberg, etc. + "hive", // provider, We support hive, jdbc-mysql, jdbc-postgresql, lakehouse-iceberg, lakehouse-paimon etc. "This is a hive catalog", hiveProperties); // Please change the properties according to the value of the provider. // ... @@ -87,13 +88,14 @@ Catalog catalog = gravitinoClient.createCatalog("catalog", Currently, Gravitino supports the following catalog providers: -| Catalog provider | Catalog property | -|---------------------|--------------------------------------------------------------------------------| -| `hive` | [Hive catalog property](./apache-hive-catalog.md#catalog-properties) | -| `lakehouse-iceberg` | [Iceberg catalog property](./lakehouse-iceberg-catalog.md#catalog-properties) | -| `jdbc-mysql` | [MySQL catalog property](./jdbc-mysql-catalog.md#catalog-properties) | -| `jdbc-postgresql` | [PostgreSQL catalog property](./jdbc-postgresql-catalog.md#catalog-properties) | -| `jdbc-doris` | [Doris catalog property](./jdbc-doris-catalog.md#catalog-properties) | +| Catalog provider | Catalog property | +|---------------------|---------------------------------------------------------------------------------| +| `hive` | [Hive catalog property](./apache-hive-catalog.md#catalog-properties) | +| `lakehouse-iceberg` | [Iceberg catalog property](./lakehouse-iceberg-catalog.md#catalog-properties) | +| `lakehouse-paimon` | [Paimon catalog property](./lakehouse-paimon-catalog.md#catalog-properties) | +| `jdbc-mysql` | [MySQL catalog property](./jdbc-mysql-catalog.md#catalog-properties) | +| `jdbc-postgresql` | [PostgreSQL catalog property](./jdbc-postgresql-catalog.md#catalog-properties) | +| `jdbc-doris` | [Doris catalog property](./jdbc-doris-catalog.md#catalog-properties) | ### Load a catalog @@ -323,6 +325,7 @@ Currently, Gravitino supports the following schema property: |---------------------|------------------------------------------------------------------------------| | `hive` | [Hive schema property](./apache-hive-catalog.md#schema-properties) | | `lakehouse-iceberg` | [Iceberg scheme property](./lakehouse-iceberg-catalog.md#schema-properties) | +| `lakehouse-paimon` | [Paimon scheme property](./lakehouse-paimon-catalog.md#schema-properties) | | `jdbc-mysql` | [MySQL schema property](./jdbc-mysql-catalog.md#schema-properties) | | `jdbc-postgresql` | [PostgreSQL schema property](./jdbc-postgresql-catalog.md#schema-properties) | | `jdbc-doris` | [Doris schema property](./jdbc-doris-catalog.md#schema-properties) | @@ -734,7 +737,7 @@ The following types that Gravitino supports: | Union | `Types.UnionType.of([type1, type2, ...])` | `{"type": "union", "types": [type JSON, ...]}` | Union type, indicates a union of types | | UUID | `Types.UUIDType.get()` | `uuid` | UUID type, indicates a universally unique identifier | -The related java doc is [here](pathname:///docs/0.5.1/api/java/org/apache/gravitino/rel/types/Type.html). +The related java doc is [here](pathname:///docs/0.6.0-incubating/api/java/org/apache/gravitino/rel/types/Type.html). ##### External type @@ -803,6 +806,7 @@ The following is a table of the column default value that Gravitino supports for |---------------------|-------------------------| | `hive` | ✘ | | `lakehouse-iceberg` | ✘ | +| `lakehouse-paimon` | ✘ | | `jdbc-mysql` | ✔ | | `jdbc-postgresql` | ✔ | @@ -815,6 +819,7 @@ The following table shows the column auto-increment that Gravitino supports for |---------------------|------------------------------------------------------------------------------| | `hive` | ✘ | | `lakehouse-iceberg` | ✘ | +| `lakehouse-paimon` | ✘ | | `jdbc-mysql` | ✔([limitations](./jdbc-mysql-catalog.md#table-column-auto-increment)) | | `jdbc-postgresql` | ✔ | @@ -826,6 +831,7 @@ The following is the table property that Gravitino supports: |---------------------|----------------------------------------------------------------------------|----------------------------------------------------------------------------| | `hive` | [Hive table property](./apache-hive-catalog.md#table-properties) | [Hive type mapping](./apache-hive-catalog.md#table-column-types) | | `lakehouse-iceberg` | [Iceberg table property](./lakehouse-iceberg-catalog.md#table-properties) | [Iceberg type mapping](./lakehouse-iceberg-catalog.md#table-column-types) | +| `lakehouse-paimon` | [Paimon table property](./lakehouse-paimon-catalog.md#table-properties) | [Paimon type mapping](./lakehouse-paimon-catalog.md#table-column-types) | | `jdbc-mysql` | [MySQL table property](./jdbc-mysql-catalog.md#table-properties) | [MySQL type mapping](./jdbc-mysql-catalog.md#table-column-types) | | `jdbc-postgresql` | [PostgreSQL table property](./jdbc-postgresql-catalog.md#table-properties) | [PostgreSQL type mapping](./jdbc-postgresql-catalog.md#table-column-types) | | `doris` | [Doris table property](./jdbc-doris-catalog.md#table-properties) | [Doris type mapping](./jdbc-doris-catalog.md#table-column-types) | @@ -836,10 +842,10 @@ In addition to the basic settings, Gravitino supports the following features: | Feature | Description | Java doc | |---------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| -| Table partitioning | Equal to `PARTITION BY` in Apache Hive, It is a partitioning strategy that is used to split a table into parts based on partition keys. Some table engine may not support this feature | [Partition](pathname:///docs/0.5.1/api/java/org/apache/gravitino/dto/rel/partitioning/Partitioning.html) | -| Table bucketing | Equal to `CLUSTERED BY` in Apache Hive, Bucketing a.k.a (Clustering) is a technique to split the data into more manageable files/parts, (By specifying the number of buckets to create). The value of the bucketing column will be hashed by a user-defined number into buckets. | [Distribution](pathname:///docs/0.5.1/api/java/org/apache/gravitino/rel/expressions/distributions/Distribution.html) | -| Table sort ordering | Equal to `SORTED BY` in Apache Hive, sort ordering is a method to sort the data in specific ways such as by a column or a function, and then store table data. it will highly improve the query performance under certain scenarios. | [SortOrder](pathname:///docs/0.5.1/api/java/org/apache/gravitino/rel/expressions/sorts/SortOrder.html) | -| Table indexes | Equal to `KEY/INDEX` in MySQL , unique key enforces uniqueness of values in one or more columns within a table. It ensures that no two rows have identical values in specified columns, thereby facilitating data integrity and enabling efficient data retrieval and manipulation operations. | [Index](pathname:///docs/0.5.1/api/java/org/apache/gravitino/rel/indexes/Index.html) | +| Table partitioning | Equal to `PARTITION BY` in Apache Hive, It is a partitioning strategy that is used to split a table into parts based on partition keys. Some table engine may not support this feature | [Partition](pathname:///docs/0.6.0-incubating/api/java/org/apache/gravitino/dto/rel/partitioning/Partitioning.html) | +| Table bucketing | Equal to `CLUSTERED BY` in Apache Hive, Bucketing a.k.a (Clustering) is a technique to split the data into more manageable files/parts, (By specifying the number of buckets to create). The value of the bucketing column will be hashed by a user-defined number into buckets. | [Distribution](pathname:///docs/0.6.0-incubating/api/java/org/apache/gravitino/rel/expressions/distributions/Distribution.html) | +| Table sort ordering | Equal to `SORTED BY` in Apache Hive, sort ordering is a method to sort the data in specific ways such as by a column or a function, and then store table data. it will highly improve the query performance under certain scenarios. | [SortOrder](pathname:///docs/0.6.0-incubating/api/java/org/apache/gravitino/rel/expressions/sorts/SortOrder.html) | +| Table indexes | Equal to `KEY/INDEX` in MySQL , unique key enforces uniqueness of values in one or more columns within a table. It ensures that no two rows have identical values in specified columns, thereby facilitating data integrity and enabling efficient data retrieval and manipulation operations. | [Index](pathname:///docs/0.6.0-incubating/api/java/org/apache/gravitino/rel/indexes/Index.html) | For more information, please see the related document on [partitioning, bucketing, sorting, and indexes](table-partitioning-bucketing-sort-order-indexes.md). @@ -980,7 +986,7 @@ There are two ways to remove a table: `dropTable` and `purgeTable`: * `dropTable` removes both the metadata and the directory associated with the table from the file system if the table is not an external table. In case of an external table, only the associated metadata is removed. * `purgeTable` completely removes both the metadata and the directory associated with the table and skipping trash, if the table is an external table or the catalogs don't support purge table, `UnsupportedOperationException` is thrown. -Hive catalog and lakehouse-iceberg catalog supports `purgeTable` while jdbc-mysql and jdbc-postgresql catalog doesn't support. +Hive catalog and lakehouse-iceberg catalog supports `purgeTable` while jdbc-mysql, jdbc-postgresql and lakehouse-paimon catalog doesn't support. ### List all tables under a schema diff --git a/docs/manage-table-partition-using-gravitino.md b/docs/manage-table-partition-using-gravitino.md index bf7f1a4c1f1..4e5ba87dd57 100644 --- a/docs/manage-table-partition-using-gravitino.md +++ b/docs/manage-table-partition-using-gravitino.md @@ -19,13 +19,13 @@ Although many catalogs inherently manage partitions automatically, there are sce The following table shows the partition operations supported across various catalogs in Gravitino: -| Operation | Hive catalog | Iceberg catalog | Jdbc-Mysql catalog | Jdbc-PostgreSQL catalog | -|-----------------------|--------------|-------------------------------------------------------------------------------|--------------------|-------------------------| -| Add Partition | ✔ | ✘ | ✘ | ✘ | -| Get Partition by Name | ✔ | ✘ | ✘ | ✘ | -| List Partition Names | ✔ | ✘ | ✘ | ✘ | -| List Partitions | ✔ | ✘ | ✘ | ✘ | -| Drop Partition | ✔ | 🚀([Coming Soon](https://github.com/apache/gravitino/issues/1655)) | ✘ | ✘ | +| Operation | Hive catalog | Iceberg catalog | Jdbc-Mysql catalog | Jdbc-PostgreSQL catalog | Jdbc-Doris catalog | +|-----------------------|--------------|-----------------------------------------------------------------------------|--------------------|-------------------------|--------------------| +| Add Partition | ✔ | ✘ | ✘ | ✘ | ✔ | +| Get Partition by Name | ✔ | ✘ | ✘ | ✘ | ✔ | +| List Partition Names | ✔ | ✘ | ✘ | ✘ | ✔ | +| List Partitions | ✔ | ✘ | ✘ | ✘ | ✔ | +| Drop Partition | ✔ | 🚀([Coming Soon](https://github.com/apache/gravitino/issues/1655)) | ✘ | ✘ | ✔ | :::tip[WELCOME FEEDBACK] If you need additional partition management support for a specific catalog, please feel free to [create an issue](https://github.com/apache/gravitino/issues/new/choose) on the [Gravitino repository](https://github.com/apache/gravitino). diff --git a/docs/manage-tags-in-gravitino.md b/docs/manage-tags-in-gravitino.md index 0f04726df1a..a02bf4faf2c 100644 --- a/docs/manage-tags-in-gravitino.md +++ b/docs/manage-tags-in-gravitino.md @@ -83,7 +83,7 @@ curl -X GET -H "Accept: application/vnd.gravitino.v1+json" \ http://localhost:8090/api/metalakes/test/tags curl -X GET -H "Accept: application/vnd.gravitino.v1+json" \ -http://localhost:8090/api/metalakes/test/tags?detailed=true +http://localhost:8090/api/metalakes/test/tags?details=true ``` diff --git a/docs/open-api/groups.yaml b/docs/open-api/groups.yaml new file mode 100644 index 00000000000..f21226b100f --- /dev/null +++ b/docs/open-api/groups.yaml @@ -0,0 +1,191 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- + +paths: + + /metalakes/{metalake}/groups: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + + post: + tags: + - access control + summary: Add group + operationId: addGroup + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/GroupAddRequest" + examples: + GroupAddRequest: + $ref: "#/components/examples/GroupAddRequest" + + responses: + "200": + description: Returns the added group object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "#/components/responses/GroupResponse" + examples: + GroupResponse: + $ref: "#/components/examples/GroupResponse" + "409": + description: Conflict - The target group already exists in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + GroupAlreadyExistsException: + $ref: "#/components/examples/GroupAlreadyExistsException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/groups/{group}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/group" + + get: + tags: + - access control + summary: Get group + operationId: getGroup + description: Returns the specified group information in the specified metalake + responses: + "200": + description: Returns the group object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "#/components/responses/GroupResponse" + examples: + GroupResponse: + $ref: "#/components/examples/GroupResponse" + "404": + description: Not Found - The specified group does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchGroupException: + $ref: "#/components/examples/NoSuchGroupException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - access control + summary: Remove group + operationId: removeGroup + responses: + "200": + $ref: "./openapi.yaml#/components/responses/RemoveResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + + schemas: + Group: + type: object + required: + - name + properties: + name: + type: string + description: The name of the group + roles: + type: array + items: + type: string + description: The roles of the group + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + GroupAddRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the group + + responses: + GroupResponse: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + group: + $ref: "#/components/schemas/Group" + + examples: + GroupAddRequest: + value: { + "name": "group1", + } + + GroupResponse: + value: { + "code": 0, + "group": { + "name": "group1", + "roles": [], + "audit": { + "creator": "gravitino", + "createTime": "2023-12-08T06:41:25.595Z" + }, + } + } + + GroupAlreadyExistsException: + value: { + "code": 1004, + "type": "GroupAlreadyExistsException", + "message": "Group already exists", + "stack": [ + "org.apache.gravitino.exceptions.GroupAlreadyExistsException: Group already exists: group1" + ] + } + + NoSuchGroupException: + value: { + "code": 1003, + "type": "NoSuchGroupException", + "message": "Group does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchGroupException: Group does not exist", + "..." + ] + } \ No newline at end of file diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index 16eb47ff955..edac71ec683 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -109,6 +109,40 @@ paths: /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics/{topic}: $ref: "./topics.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1topics~1%7Btopic%7D" + + /metalakes/{metalake}/users: + $ref: "./users.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1users" + + /metalakes/{metalake}/users/{user}: + $ref: "./users.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1users~1%7Buser%7D" + + /metalakes/{metalake}/groups: + $ref: "./groups.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1groups" + + /metalakes/{metalake}/groups/{group}: + $ref: "./groups.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1groups~1%7Bgroup%7D" + + /metalakes/{metalake}/roles: + $ref: "./roles.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1roles" + + /metalakes/{metalake}/roles/{role}: + $ref: "./roles.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1roles~1%7Brole%7D" + + /metalakes/{metalake}/owners/{metadataObjectType}/{metadataObjectFullName}: + $ref: "./owners.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1owners~1%7BmetadataObjectType%7D~1%7BmetadataObjectFullName%7D" + + /metalakes/{metalake}/permissions/users/{user}/grant: + $ref: "./permissions.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1permissions~1users~1%7Buser%7D~1grant" + + /metalakes/{metalake}/permissions/users/{user}/revoke: + $ref: "./permissions.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1permissions~1users~1%7Buser%7D~1revoke" + + /metalakes/{metalake}/permissions/groups/{group}/grant: + $ref: "./permissions.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1permissions~1groups~1%7Bgroup%7D~1grant" + + /metalakes/{metalake}/permissions/groups/{group}/revoke: + $ref: "./permissions.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1permissions~1groups~1%7Bgroup%7D~1revoke" + components: schemas: @@ -254,6 +288,57 @@ components: type: boolean description: Whether the drop operation was successful + RemoveResponse: + description: Represents a response for a remove operation + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + removed: + type: boolean + description: Whether the remove operation was successful + + DeleteResponse: + description: Represents a response for a delete operation + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + deleted: + type: boolean + description: Whether the delete operation was successful + + SetResponse: + description: Represents a response for a set operation + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + set: + type: boolean + description: Whether the set operation was successful + parameters: metalake: name: metalake @@ -311,6 +396,29 @@ components: schema: type: string + user: + name: user + in: path + description: The name of the user + required: true + schema: + type: string + + group: + name: group + in: path + description: The name of the group + required: true + schema: + type: string + role: + name: role + in: path + description: The name of the role + required: true + schema: + type: string + metadataObjectType: name: metadataObjectType in: path @@ -324,6 +432,8 @@ components: - "table" - "fileset" - "topic" + - "role" + - "metalake" metadataObjectFullName: name: metadataObjectFullName diff --git a/docs/open-api/owners.yaml b/docs/open-api/owners.yaml new file mode 100644 index 00000000000..9514a0a12ed --- /dev/null +++ b/docs/open-api/owners.yaml @@ -0,0 +1,176 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- + +paths: + + /metalakes/{metalake}/owners/{metadataObjectType}/{metadataObjectFullName}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/metadataObjectType" + - $ref: "./openapi.yaml#/components/parameters/metadataObjectFullName" + + put: + tags: + - access control + summary: Set owner + operationId: setOwner + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OwnerSetRequest" + examples: + OwnerSetRequest: + $ref: "#/components/examples/OwnerSetRequest" + + responses: + "200": + description: Returns the set operation result + $ref: "./openapi.yaml#/components/responses/SetResponse" + + "404": + description: Not Found - The specified owner or metadata object does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NotFoundException: + $ref: "#/components/examples/NotFoundException" + + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + get: + tags: + - access control + summary: Get owner + operationId: getOwner + description: Returns the specified owner information in the specified metalake + responses: + "200": + description: Returns the owner object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "#/components/responses/OwnerResponse" + examples: + OwnerResponse: + $ref: "#/components/examples/OwnerResponse" + "404": + description: Not Found - The specified owner does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetadataObjectException: + $ref: "#/components/examples/NoSuchMetadataObjectException" + + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + +components: + + schemas: + Owner: + type: object + required: + - name + - type + properties: + name: + type: string + description: The name of the owner + type: + type: string + enum: + - USER + - GROUP + description: The type of the owner + + OwnerSetRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the owner + type: + type: string + enum: + - USER + - GROUP + description: The type of the owner + + responses: + OwnerResponse: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + owner: + $ref: "#/components/schemas/Owner" + + examples: + OwnerSetRequest: + value: { + "name": "user1", + "type": "USER" + } + + OwnerResponse: + value: { + "code": 0, + "owner": { + "name": "user1", + "type": "USER", + } + } + + + NoSuchMetadataObjectException: + value: { + "code": 1003, + "type": "NoSuchMetadataObjectException", + "message": "Metadata object does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchUserException: Metadata object does not exist", + "..." + ] + } + + NotFoundException: + value: { + "code": 1003, + "type": "NotFoundException", + "message": "Metadata object or owner does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NotFoundException: Metadata object or owner does not exist", + "..." + ] + } \ No newline at end of file diff --git a/docs/open-api/permissions.yaml b/docs/open-api/permissions.yaml new file mode 100644 index 00000000000..0b5bbddd7e7 --- /dev/null +++ b/docs/open-api/permissions.yaml @@ -0,0 +1,262 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- + +paths: + + /metalakes/{metalake}/permissions/users/{user}/grant: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/user" + + put: + tags: + - access control + summary: Grant roles to a user + operationId: grantRoleToUser + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RoleGrantRequest" + examples: + RoleGrantRequest: + $ref: "#/components/examples/RoleGrantRequest" + + responses: + "200": + description: Returns the granted user object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./users.yaml#/components/responses/UserResponse" + examples: + UserResponse: + $ref: "./users.yaml#/components/examples/UserResponse" + + "404": + description: Not Found - The specified user or role does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchUserException: + $ref: "./users.yaml#/components/examples/NoSuchUserException" + NoSuchRoleException: + $ref: "./roles.yaml#/components/examples/NoSuchRoleException" + + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/permissions/users/{user}/revoke: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/user" + + put: + tags: + - access control + summary: Revoke roles to a user + operationId: revokeRoleFromUser + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RoleRevokeRequest" + examples: + RoleRevokeRequest: + $ref: "#/components/examples/RoleRevokeRequest" + + responses: + "200": + description: Returns the revoked user object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./users.yaml#/components/responses/UserResponse" + examples: + UserResponse: + $ref: "./users.yaml#/components/examples/UserResponse" + + "404": + description: Not Found - The specified user or role does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchUserException: + $ref: "./users.yaml#/components/examples/NoSuchUserException" + NoSuchRoleException: + $ref: "./roles.yaml#/components/examples/NoSuchRoleException" + + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/permissions/groups/{group}/grant: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/group" + + put: + tags: + - access control + summary: Grant roles to a group + operationId: grantRoleToGroup + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RoleGrantRequest" + examples: + RoleGrantRequest: + $ref: "#/components/examples/RoleGrantRequest" + + responses: + "200": + description: Returns the granted group object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./groups.yaml#/components/responses/GroupResponse" + examples: + GroupResponse: + $ref: "./groups.yaml#/components/examples/GroupResponse" + + "404": + description: Not Found - The specified group or role does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchGroupException: + $ref: "./groups.yaml#/components/examples/NoSuchGroupException" + NoSuchRoleException: + $ref: "./roles.yaml#/components/examples/NoSuchRoleException" + + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/permissions/groups/{group}/revoke: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/group" + + put: + tags: + - access control + summary: Revoke roles to a group + operationId: revokeRoleFromGroup + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RoleRevokeRequest" + examples: + RoleRevokeRequest: + $ref: "#/components/examples/RoleRevokeRequest" + + responses: + "200": + description: Returns the revoked group object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./groups.yaml#/components/responses/GroupResponse" + examples: + GroupResponse: + $ref: "./groups.yaml#/components/examples/GroupResponse" + + "404": + description: Not Found - The specified group or role does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchGroupException: + $ref: "./groups.yaml#/components/examples/NoSuchGroupException" + NoSuchRoleException: + $ref: "./roles.yaml#/components/examples/NoSuchRoleException" + + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + + schemas: + User: + type: object + required: + - name + properties: + name: + type: string + description: The name of the user + roles: + type: array + items: + type: string + description: The roles of the user + + RoleGrantRequest: + type: object + required: + - roleNames + properties: + roleNames: + type: array + description: The role names need to grant + items: + type: string + + RoleRevokeRequest: + type: object + required: + - roleNames + properties: + roleNames: + type: array + description: The role names need to revoke + items: + type: string + + + examples: + + RoleGrantRequest: + value: { + "roleNames": ["role1"], + } + + RoleRevokeRequest: + value: { + "roleNames": [ "role1" ], + } \ No newline at end of file diff --git a/docs/open-api/roles.yaml b/docs/open-api/roles.yaml new file mode 100644 index 00000000000..005cd89e3cb --- /dev/null +++ b/docs/open-api/roles.yaml @@ -0,0 +1,311 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- + +paths: + + /metalakes/{metalake}/roles: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + + post: + tags: + - access control + summary: Create role + operationId: createRole + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RoleCreateRequest" + examples: + RoleCreateRequest: + $ref: "#/components/examples/RoleCreateRequest" + + responses: + "200": + description: Returns the created role object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "#/components/responses/RoleResponse" + examples: + RoleResponse: + $ref: "#/components/examples/RoleResponse" + + "404": + description: Not Found - The specified securable object does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetadataObjectException: + $ref: "#/components/examples/NoSuchMetadataObjectException" + + "409": + description: Conflict - The target role already exists in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + RoleAlreadyExistsException: + $ref: "#/components/examples/RoleAlreadyExistsException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/roles/{role}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/role" + + get: + tags: + - access control + summary: Get Role + operationId: getRole + description: Returns the specified role information in the specified metalake + responses: + "200": + description: Returns the role object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "#/components/responses/RoleResponse" + examples: + RoleResponse: + $ref: "#/components/examples/RoleResponse" + "404": + description: Not Found - The specified role does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchRoleException: + $ref: "#/components/examples/NoSuchRoleException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - access control + summary: Delete role + operationId: deleteRole + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DeleteResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + + schemas: + + Privilege: + type: object + required: + - name + - condition + properties: + name: + type: string + enum: + - CREATE_CATALOG + - USE_CATALOG + - CREATE_SCHEMA + - USE_SCHEMA + - CREATE_TABLE + - MODIFY_TABLE + - SELECT_TABLE + - CREATE_FILESET + - WRITE_FILESET + - READ_FILESET + - CREATE_TOPIC + - PRODUCE_TOPIC + - CONSUME_TOPIC + - MANAGE_USERS + - MANAGE_GROUPS + - CREATE_ROLE + - MANAGE_GRANTS + description: The name of the privilege + condition: + type: string + enum: + - ALLOW + - DENY + description: The condition of the privilege, `ALLOW` or `DENY` + + SecurableObject: + type: object + required: + - fullName + - type + properties: + fullName: + type: string + description: The full name of the securable object + type: + type: string + enum: + - "CATALOG" + - "SCHEMA" + - "TABLE" + - "FILESET" + - "TOPIC" + - "METALAKE" + description: The type of the securable object + privileges: + type: array + description: A list of privileges + items: + $ref: "#/components/schemas/Privilege" + + Role: + type: object + required: + - name + properties: + name: + type: string + description: The name of the role + properties: + type: object + description: A map of properties for the role + nullable: true + default: { } + additionalProperties: + type: string + securableObjects: + type: array + description: A list of securable objects + items: + $ref: "#/components/schemas/SecurableObject" + + RoleCreateRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the role + properties: + type: object + description: A map of properties for the role + nullable: true + default: { } + additionalProperties: + type: string + securableObjects: + type: array + description: A list of securable objects + items: + $ref: "#/components/schemas/SecurableObject" + + responses: + RoleResponse: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + role: + $ref: "#/components/schemas/Role" + + examples: + RoleCreateRequest: + value: { + "name": "role1", + "properties": {"k1": "v1"}, + "securableObjects": [ + { + "fullName" : "catalog1.schema1.table1", + "type": "TABLE", + "privileges": [ + { + "name": "SELECT_TABLE", + "condition": "ALLOW" + } + ] + } + ] + } + + RoleResponse: + value: { + "code": 0, + "role": { + "name": "role1", + "properties" : { "k1": "v1" }, + "securableObjects": [ + { + "fullName": "catalog1.schema1.table1", + "type": "TABLE", + "privileges": [ + { + name: "SELECT_TABLE", + condition: "ALLOW" + } + ] + } + ] + } + } + + RoleAlreadyExistsException: + value: { + "code": 1004, + "type": "RoleAlreadyExistsException", + "message": "Role already exists", + "stack": [ + "org.apache.gravitino.exceptions.RoleAlreadyExistsException: Role already exists: role1" + ] + } + + NoSuchRoleException: + value: { + "code": 1003, + "type": "NoSuchRoleException", + "message": "Role does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchRoleException: Role does not exist", + "..." + ] + } + + NoSuchMetadataObjectException: + value: { + "code": 1003, + "type": "NoSuchMetadataObjectException", + "message": "Metadata object does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchUserException: Metadata object does not exist", + "..." + ] + } \ No newline at end of file diff --git a/docs/open-api/tags.yaml b/docs/open-api/tags.yaml index 51d211c0999..61d9ae1fc84 100644 --- a/docs/open-api/tags.yaml +++ b/docs/open-api/tags.yaml @@ -188,7 +188,7 @@ paths: get: tags: - tag - summary: List tags (names) associated with the specified metadata object + summary: List tags (names) for metadata object operationId: listObjectTags parameters: - $ref: "#/components/parameters/details" @@ -214,7 +214,7 @@ paths: post: tags: - tag - summary: Associate tags with the specified metadata object + summary: Associate tags with metadata object operationId: associateTags requestBody: content: @@ -256,7 +256,7 @@ paths: get: tags: - tag - summary: get associated tag for metadata object with specified tag name + summary: Get tag for metadata object operationId: getObjectTag parameters: - $ref: "#/components/parameters/details" diff --git a/docs/open-api/users.yaml b/docs/open-api/users.yaml new file mode 100644 index 00000000000..d83c27a4663 --- /dev/null +++ b/docs/open-api/users.yaml @@ -0,0 +1,191 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- + +paths: + + /metalakes/{metalake}/users: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + + post: + tags: + - access control + summary: Add user + operationId: addUser + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/UserAddRequest" + examples: + UserAddRequest: + $ref: "#/components/examples/UserAddRequest" + + responses: + "200": + description: Returns the added user object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "#/components/responses/UserResponse" + examples: + UserResponse: + $ref: "#/components/examples/UserResponse" + "409": + description: Conflict - The target user already exists in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + UserAlreadyExistsException: + $ref: "#/components/examples/UserAlreadyExistsException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/users/{user}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/user" + + get: + tags: + - access control + summary: Get user + operationId: getUser + description: Returns the specified user information in the specified metalake + responses: + "200": + description: Returns the user object + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "#/components/responses/UserResponse" + examples: + UserResponse: + $ref: "#/components/examples/UserResponse" + "404": + description: Not Found - The specified user does not exist in the specified metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchUserException: + $ref: "#/components/examples/NoSuchUserException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - access control + summary: Remove user + operationId: removeUser + responses: + "200": + $ref: "./openapi.yaml#/components/responses/RemoveResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + + schemas: + User: + type: object + required: + - name + properties: + name: + type: string + description: The name of the user + roles: + type: array + items: + type: string + description: The roles of the user + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + UserAddRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the user + + responses: + UserResponse: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + user: + $ref: "#/components/schemas/User" + + examples: + UserAddRequest: + value: { + "name": "user1", + } + + UserResponse: + value: { + "code": 0, + "user": { + "name": "user1", + "roles": [], + "audit": { + "creator": "gravitino", + "createTime": "2023-12-08T06:41:25.595Z" + }, + } + } + + UserAlreadyExistsException: + value: { + "code": 1004, + "type": "UserAlreadyExistsException", + "message": "User already exists", + "stack": [ + "org.apache.gravitino.exceptions.UserAlreadyExistsException: User already exists: user1" + ] + } + + NoSuchUserException: + value: { + "code": 1003, + "type": "NoSuchUserException", + "message": "User does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchUserException: User does not exist", + "..." + ] + } \ No newline at end of file diff --git a/docs/security.md b/docs/security.md deleted file mode 100644 index 190a9094140..00000000000 --- a/docs/security.md +++ /dev/null @@ -1,326 +0,0 @@ ---- -title: "Security" -slug: /security -keyword: security -license: "This software is licensed under the Apache License version 2." ---- - -## Authentication - -Apache Gravitino supports two kinds of authentication mechanisms: simple and OAuth. - -### Simple mode - -Simple mode is the default authentication option of the server. - -For the client side, if it doesn't set the authentication explicitly, it will use anonymous to access the server. - -If the client sets the simple mode, it will use the environment variable `GRAVITINO_USER` as the user. - -If the environment variable `GRAVITINO_USER` isn't set, the client uses the user of the machine that sends requests. - -For the client side, users can enable `simple` mode by the following code: - -```java -GravitinoClient client = GravitinoClient.builder(uri) - .withMetalake("metalake") - .withSimpleAuth() - .build(); -``` - -### OAuth mode - -Gravitino only supports external OAuth 2.0 servers. - -First, users need to guarantee that the external correctly configured OAuth 2.0 server supports Bearer JWT. - -Then, on the server side, users should set `gravitino.authenticator` as `oauth` and give -`gravitino.authenticator.oauth.defaultSignKey`, `gravitino.authenticator.oauth.serverUri` and -`gravitino.authenticator.oauth.tokenPath` a proper value. - -Next, for the client side, users can enable `OAuth` mode by the following code: - -```java -DefaultOAuth2TokenProvider authDataProvider = DefaultOAuth2TokenProvider.builder() - .withUri("oauth server uri") - .withCredential("yy:xx") - .withPath("oauth/token") - .withScope("test") - .build(); - -GravitinoClient client = GravitinoClient.builder(uri) - .withMetalake("metalake") - .withOAuth(authDataProvider) - .build(); -``` - -### Kerberos mode - -Gravitino supports Kerberos mode. - -For the server side, users should set `gravitino.authenticator` as `kerberos` and give -`gravitino.authenticator.kerberos.principal` and `gravitino.authenticator.kerberos.keytab` a proper value. - -For the client side, users can enable `kerberos` mode by the following code: - -```java -// Use keytab to create KerberosTokenProvider -KerberosTokenProvider provider = KerberosTokenProvider.builder() - .withClientPrincipal(clientPrincipal) - .withKeyTabFile(new File(keytabFile)) - .build(); - -// Use ticketCache to create KerberosTokenProvider -KerberosTokenProvider provider = KerberosTokenProvider.builder() - .withClientPrincipal(clientPrincipal) - .build(); - -GravitinoClient client = GravitinoClient.builder(uri) - .withMetalake("metalake") - .withKerberosAuth(provider) - .build(); -``` - -:::info -Now Iceberg REST service doesn't support Kerberos authentication. -The URI must use the hostname of server instead of IP. -::: - -### Server configuration - -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------------|-----------------------------------------------------------------------------------------------------|-------------------|--------------------------------------------|---------------| -| `gravitino.authenticator` | The authenticator which Gravitino uses, setting as `simple`,`oauth` or `kerberos`. | `simple` | No | 0.3.0 | -| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | No | 0.3.0 | -| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | No | 0.3.0 | -| `gravitino.authenticator.oauth.defaultSignKey` | The signing key of JWT when Gravitino uses OAuth as the authenticator. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | No | 0.3.0 | -| `gravitino.authenticator.oauth.serverUri` | The URI of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.oauth.tokenPath` | The path for token of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.kerberos.principal` | Indicates the Kerberos principal to be used for HTTP endpoint. Principal should start with `HTTP/`. | (none) | Yes if use `kerberos` as the authenticator | 0.4.0 | -| `gravitino.authenticator.kerberos.keytab` | Location of the keytab file with the credentials for the principal. | (none) | Yes if use `kerberos` as the authenticator | 0.4.0 | - -The signature algorithms that Gravitino supports follows: - -| Name | Description | -|-------|------------------------------------------------| -| HS256 | HMAC using SHA-25A | -| HS384 | HMAC using SHA-384 | -| HS512 | HMAC using SHA-51 | -| RS256 | RSASSA-PKCS-v1_5 using SHA-256 | -| RS384 | RSASSA-PKCS-v1_5 using SHA-384 | -| RS512 | RSASSA-PKCS-v1_5 using SHA-512 | -| ES256 | ECDSA using P-256 and SHA-256 | -| ES384 | ECDSA using P-384 and SHA-384 | -| ES512 | ECDSA using P-521 and SHA-512 | -| PS256 | RSASSA-PSS using SHA-256 and MGF1 with SHA-256 | -| PS384 | RSASSA-PSS using SHA-384 and MGF1 with SHA-384 | -| PS512 | RSASSA-PSS using SHA-512 and MGF1 with SHA-512 | - -### Example - -You can follow the steps to set up an OAuth mode Gravitino server. - -1. Prerequisite - - You need to install the JDK8 and Docker. - -2. Set up an external OAuth 2.0 server - - There is a sample-authorization-server based on [spring-authorization-server](https://github.com/spring-projects/spring-authorization-server/tree/1.0.3). - - The image has registered client information in the external OAuth 2.0 server. - - Its clientId is `test`. Its secret is `test`. Its scope is `test`. - -```shell - docker run -p 8177:8177 --name sample-auth-server -d datastrato/sample-authorization-server:0.3.0 -``` - -3. Open [the JWK URL of the Authorization server](http://localhost:8177/oauth2/jwks) in the browser and you can get the JWK. - - ![jks_response_image](assets/jks.png) - -4. Convert the JWK to PEM. You can use the [online tool](https://8gwifi.org/jwkconvertfunctions.jsp#google_vignette) or other tools. - - ![pem_convert_result_image](assets/pem.png) - -5. Copy the public key and remove the character `\n` and you can get the default signing key of Gravitino server. - -6. You can refer to the [Configurations](gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. - -```text -gravitino.authenticator = oauth -gravitino.authenticator.oauth.serviceAudience = test -gravitino.authenticator.oauth.defaultSignKey = -gravitino.authenticator.oauth.tokenPath = /oauth2/token -gravitino.authenticator.oauth.serverUri = http://localhost:8177 -``` - -7. Open [the URL of Gravitino server](http://localhost:8090) and login in with clientId `test`, clientSecret `test`, and scope `test`. - - ![oauth_login_image](assets/oauth.png) - -8. You can also use the curl command to access Gravitino. - -Get access token - -```shell -curl --location --request POST 'http://127.0.0.1:8177/oauth2/token?grant_type=client_credentials&client_id=test&client_secret=test&scope=test' -``` - -Use the access token to request the Gravitino - -```shell -curl -v -X GET -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" -H "Authorization: Bearer " http://localhost:8090/api/version -``` - -## HTTPS - -Users would better use HTTPS instead of HTTP if users choose OAuth 2.0 as the authenticator. - -HTTPS protects the header of the request from smuggling, making it safer. - -If users choose to enable HTTPS, Gravitino won't provide the ability of HTTP service. - -Both the Gravitino server and Iceberg REST service can configure HTTPS. - -### Apache Gravitino server's configuration - -| Configuration item | Description | Default value | Required | Since version | -|-----------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | -| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | -| `gravitino.server.webserver.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | -| `gravitino.server.webserver.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | -| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | -| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | -| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | - -### Apache Iceberg REST service's configuration - -| Configuration item | Description | Default value | Required | Since version | -|-------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | -| `gravitino.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `9433` | No | 0.3.0 | -| `gravitino.iceberg-rest.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.iceberg-rest.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | -| `gravitino.iceberg-rest.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | -| `gravitino.iceberg-rest.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | -| `gravitino.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | -| `gravitino.iceberg-rest.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.iceberg-rest.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | - -Refer to the "Additional JSSE Standard Names" section of the [Java security guide](https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#jssenames) for the list of protocols related to tlsProtocol. You can find the list of `tlsProtocol` values for Java 8 in this document. - -Refer to the "Additional JSSE Standard Names" section of the [Java security guide](https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#ciphersuites) for the list of protocols related to tlsProtocol. You can find the list of `enableCipherAlgorithms` values for Java 8 in this document. - -### Example - -You can follow the steps to set up an HTTPS server. - -1. Prerequisite - - You need to install the JDK8, wget, and set the environment JAVA_HOME. - - If you want to use the command `curl` to request the Gravitino server, you should install openSSL. - -2. Generate the key store - -```shell -cd $JAVA_HOME -bin/keytool -genkeypair -alias localhost \ --keyalg RSA -keysize 4096 -keypass localhost \ --sigalg SHA256withRSA \ --keystore localhost.jks -storetype JKS -storepass localhost \ --dname "cn=localhost,ou=localhost,o=localhost,l=beijing,st=beijing,c=cn" \ --validity 36500 -``` - -3. Generate the certificate - -```shell -bin/keytool -export -alias localhost -keystore localhost.jks -file localhost.crt -storepass localhost -``` - -4. Import the certificate - -```shell -bin/keytool -import -alias localhost -keystore jre/lib/security/cacerts -file localhost.crt -storepass changeit -noprompt -``` - -5. You can refer to the [Configurations](gravitino-server-config.md) and append the configuration to the conf/gravitino.conf. -Configuration doesn't support resolving environment variables, so you should replace `${JAVA_HOME}` with the actual value. -Then, You can start the Gravitino server. - -```text -gravitino.server.webserver.host = localhost -gravitino.server.webserver.enableHttps = true -gravitino.server.webserver.keyStorePath = ${JAVA_HOME}/localhost.jks -gravitino.server.webserver.keyStorePassword = localhost -gravitino.server.webserver.managerPassword = localhost -``` - -6. Request the Gravitino server - - If you use Java, you can follow the steps - - Copy the code to a file named Main.java - -```java -import org.apache.gravitino.client.GravitinoClient; -import org.apache.gravitino.client.GravitinoVersion; - -public class Main { - public static void main(String[] args) { - String uri = "https://localhost:8433"; - GravitinoClient client = GravitinoClient.builder(uri).withMetalake("metalake").build(); - GravitinoVersion gravitinoVersion = client.getVersion(); - System.out.println(gravitinoVersion); - } -} -``` - -If you want to use the command `curl`, you can follow the commands: - -```shell -openssl x509 -inform der -in $JAVA_HOME/localhost.crt -out certificate.pem -curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" https://localhost:8433/api/version -``` -## Cross-origin resource filter - -### Server configuration - -| Configuration item | Description | Default value | Required | Since version | -|----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------|----------|---------------| -| `gravitino.server.webserver.enableCorsFilter` | Enable cross-origin resource share filter. | false | No | 0.4.0 | -| `gravitino.server.webserver.allowedOrigins` | A comma separated list of allowed origins to access the resources. The default value is *, which means all origins. | `*` | No | 0.4.0 | -| `gravitino.server.webserver.allowedTimingOrigins` | A comma separated list of allowed origins to time the resource. The default value is the empty string, which means no origins. | `` | No | 0.4.0 | -| `gravitino.server.webserver.allowedMethods` | A comma separated list of allowed HTTP methods used when accessing the resources. The default values are GET, POST, HEAD, and DELETE. | `GET,POST,HEAD,DELETE,PUT` | No | 0.4.0 | -| `gravitino.server.webserver.allowedHeaders` | A comma separated list of allowed HTTP headers specified when accessing the resources. The default value is X-Requested-With,Content-Type,Accept,Origin. If the value is a single *, it accepts all headers. | `X-Requested-With,Content-Type,Accept,Origin` | No | 0.4.0 | -| `gravitino.server.webserver.preflightMaxAgeInSecs` | The number of seconds to cache preflight requests by the client. The default value is 1800 seconds or 30 minutes. | `1800` | No | 0.4.0 | -| `gravitino.server.webserver.allowCredentials` | A boolean indicating if the resource allows requests with credentials. The default value is true. | `true` | No | 0.4.0 | -| `gravitino.server.webserver.exposedHeaders` | A comma separated list of allowed HTTP headers exposed on the client. The default value is the empty list. | `` | No | 0.4.0 | -| `gravitino.server.webserver.chainPreflight` | If true chained preflight requests for normal handling (as an OPTION request). Otherwise, the filter responds to the preflight. The default is true. | `true` | No | 0.4.0 | - -### Apache Iceberg REST service's configuration - -| Configuration item | Description | Default value | Required | Since version | -|------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------|----------|---------------| -| `gravitino.iceberg-rest.enableCorsFilter` | Enable cross-origin resource share filter. | false | No | 0.4.0 | -| `gravitino.iceberg-rest.allowedOrigins` | A comma separated list of allowed origins that access the resources. The default value is *, which means all origins. | `*` | No | 0.4.0 | -| `gravitino.iceberg-rest.allowedTimingOrigins` | A comma separated list of allowed origins that time the resource. The default value is the empty string, which means no origins. | `` | No | 0.4.0 | -| `gravitino.iceberg-rest.allowedMethods` | A comma separated list of allowed HTTP methods used when accessing the resources. The default values are GET, POST, HEAD, and DELETE. | `GET,POST,HEAD,DELETE,PUT` | No | 0.4.0 | -| `gravitino.iceberg-rest.allowedHeaders` | A comma separated list of HTTP allowed headers specified when accessing the resources. The default value is X-Requested-With,Content-Type,Accept,Origin. If the value is a single *, it accepts all headers. | `X-Requested-With,Content-Type,Accept,Origin` | No | 0.4.0 | -| `gravitino.iceberg-rest.preflightMaxAgeInSecs` | The number of seconds to cache preflight requests by the client. The default value is 1800 seconds or 30 minutes. | `1800` | No | 0.4.0 | -| `gravitino.iceberg-rest.allowCredentials` | A boolean indicating if the resource allows requests with credentials. The default value is true. | `true` | No | 0.4.0 | -| `gravitino.iceberg-rest.exposedHeaders` | A comma separated list of allowed HTTP headers exposed on the client. The default value is the empty list. | `` | No | 0.4.0 | -| `gravitino.iceberg-rest.chainPreflight` | If true chained preflight requests for normal handling (as an OPTION request). Otherwise, the filter responds to the preflight. The default is true. | `true` | No | 0.4.0 | diff --git a/docs/security/access-control.md b/docs/security/access-control.md new file mode 100644 index 00000000000..902fc059167 --- /dev/null +++ b/docs/security/access-control.md @@ -0,0 +1,640 @@ +--- +title: "Access Control" +slug: /security/access-control +keyword: security +license: "This software is licensed under the Apache License version 2." +--- + +## Overview + +Gravitino adopts RBAC and DAC. + +Role-based Access Control (RBAC): Access privileges are assigned to roles, which are in turn assigned to users or groups. + +Discretionary Access Control(DAC): Each metadata object has an owner, who can in turn grant access to that object. + +:::info + +Gravitino only supports authorization for securable objects, when Gravitino supports to pass the privileges to underlying authorization plugin. +Gravitino doesn't support metadata authentication. It means that Gravitino won't check the privileges when Gravitino receives the requests. + +::: + + +## Concept + +### Role + +A metadata object to which privileges can be granted. Roles are in turn assigned to users or groups. + +### Privilege + +A defined level of access to an object. Multiple distinct privileges may be used to control the granularity of access granted. + +### User + +A user identity recognized by Gravitino. External user system instead of Gravitino manages users. + +### Group + +A group identity recognized by Gravitino. External user system instead of Gravitino manages groups. + +### Metadata objects + +Metadata objects are managed in Gravitino, such as `CATALOG`, `SCHEMA`, `TABLE`, +`COLUMN`, `FILESET`, `TOPIC`, `COLUMN`, `ROLE`, `METALAKE`. A metadata object is combined by a `type` and a +comma-separated `name`. For example, a `CATAGLOG` object has a name "catalog1" with type +"CATALOG", a `SCHEMA` object has a name "catalog1.schema1" with type "SCHEMA", a `TABLE` +object has a name "catalog1.schema1.table1" with type "TABLE". A `METALAKE` object has a name "metalake1". + +### Securable objects + +A metadata object to which access can be granted. Unless allowed by a grant, access is denied. +Every securable object resides within a logical container in a hierarchy of containers. +The top container is the metalake. +Catalogs are under the metalake. Catalogs represent different kinds of data sources. +Schemas are under the catalog. There are tables, topics, or filesets under the schema. + +![object_image](../assets/object.png) + +The relationship of the concepts is as below. + +![user_group_relationship_image](../assets/user-group.png) +![concept_relationship_image](../assets/role.png) + +### Ownership + +Every metadata object has an owner. The owner could be a user or group, and has all the privileges of the metadata object. +Meanwhile, you can transfer the ownership of securable object to another user or group. + +## The types of roles + +### Service Admin + +Service admin is only used for managing the metalakes. Usually, this role is for the maintainer of the service. + +### Custom Roles + +You can also create a dedicated role for your business by API or the client. + +## The types of privileges + +### User privileges + +| Name | Supports Securable Object | Operation | +|-------------|---------------------------|---------------------| +| ManageUsers | Metalake | Add or remove users | + +### Group privileges + +| Name | Supports Securable Object | Operation | +|--------------|---------------------------|----------------------| +| ManageGroups | Metalake | Add or remove groups | + +### Role privileges + +| Name | Supports Securable Object | Operation | +|------------|---------------------------|---------------| +| CreateRole | Metalake | Create a role | + +### Permission privileges + +| Name | Supports Securable Object | Operation | +|--------------|---------------------------|------------------------| +| ManageGrants | Metalake | grant or revoke a role | + +### Catalog privileges + +| Name | Supports Securable Object | Operation | +|---------------|---------------------------|------------------| +| CreateCatalog | Metalake | Create a catalog | +| UseCatalog | Metalake, Catalog | | + +:::info + +`USE_CATALOG` is needed for a user to interact with any object within the catalog. + +For example, to select data from a table, users need to have the SELECT_TABLE privilege on that table and +`USE CATALOG` privileges on its parent catalog as well as `USE SCHEMA` privileges on its parent schema. + +::: + +### Schema privileges + +| Name | Supports Securable Object | Operation | +|--------------|---------------------------|-----------------| +| CreateSchema | Metalake, Catalog | Create a schema | +| UseSchema | Metalake, Catalog, Schema | Use a schema | + +:::info + +`UseSchema`is needed for a user to interact with any object within the schema. + +For example, to select data from a table, users need to have the `SELECT_TABLE` privilege on that table +and `USE SCHEMA` privileges on its parent schema. + +::: + +### Table privileges + +| Name | Supports Securable Object | Operation | +|-------------|-----------------------------------|------------------------------------------------| +| CreateTable | Metalake, Catalog, Schema | Create a table | +| ModifyTable | Metalake, Catalog, Schema, Table | Use the SQL `UPDATE`,`DELETE`,`INSERT` a table | +| SelectTable | Metalake, Catalog, Schema, Table | Use the SQL `SELECT` data from a table | + +### Topic privileges + +| Name | Supports Securable Object | Operation | +|--------------|----------------------------------|-------------------------------------------| +| CreateTopic | Metalake, Catalog, Schema | Create a topic | +| ProduceTopic | Metalake, Catalog, Schema, Topic | Produce a topic (including alter a topic) | +| ConsumeTopic | Metalake, Catalog, Schema, Topic | Consume a topic | + +### Fileset privileges + +| Name | Supports Securable Object | Operation | +|---------------|------------------------------------|---------------------------------------------| +| CreateFileset | Metalake, Catalog, Schema | Create a fileset | +| WriteFileset | Metalake, Catalog, Schema, Fileset | Write a fileset (including alter a fileset) | +| ReadFileset | Metalake, Catalog, Schema, Fileset | read a fileset | + +## Inheritance Model + +Securable objects in Gravitino are hierarchical and privileges are inherited downward. + +This means that granting a privilege on a metalake, catalog or schema automatically grants +the privilege to all current and future objects within the metalake, catalog or schema. + +For example, if you give a use that `SELECT_TABLE` privilege on a catalog, then that the user +will be able to select(read) all tables in that catalog. + +## Privilege Condition + +The privilege supports two condition: `allow` and `deny`. `allow` means that you are able to use the privilege, +`deny` means that you aren't able to use the privilege. +`deny` condition is prior to `allow` condition. If a role has the `allow` condition and `deny` condition at the same time. +The user won't be able to use the privilege. + +If parent securable object has the same privilege name with different condition, the securable object won't override the parent object privilege. +For example, securable metalake object allows to use the catalog, but securable catalog denies to use the catalog, the user isn't able to use the catalog. +If securable metalake object denies to use the catalog, but securable catalog allows to use the catalog, the user isn't able to use the catalog, too. + +![privilege_image](../assets/privilege.png) + +## Server Configuration + +If you want to enable the access control, you should enable the authorization. + +The related configuration is as follows. + +| Configuration item | Description | Default value | Required | Since Version | +|------------------------------------------|------------------------------------------------------------------------|---------------|----------------------------------|---------------| +| `gravitino.authorization.enable` | Whether Gravitino enable authorization or not. | false | No | 0.5.0 | +| `gravitino.authorization.serviceAdmins` | The admins of Gravitino service, Multiple admins are spitted by comma. | (none) | Yes if enables the authorization | 0.5.0 | + + +## User Operation + +### Add a user + +You should add the user to your metalake before you use the authorization. + + + + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "user1", +}' http://localhost:8090/api/metalakes/test/users +``` + + + + +```java +GravitinoClient client = ... +User user = + client.addUser("user1"); +``` + + + + +### Get a user + +You can get a user by its name. + + + + +```shell +curl -X GET -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" http://localhost:8090/api/metalakes/test/users/user1 +``` + + + + +```java +GravitinoClient client = ... +User user = + client.getUser("user1"); +``` + + + + +### Delete a user + +You can delete a user by its name. + + + + +```shell +curl -X DELETE -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" http://localhost:8090/api/metalakes/test/users/user1 +``` + + + + +```java +GravitinoClient client = ... +boolean deleted = + client.deleteUser("user1"); +``` + + + + +## Group Operation + +### Add a Group + +You should add the group to your metalake before you use the authorization. + + + + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "group1", +}' http://localhost:8090/api/metalakes/test/groups +``` + + + + +```java +GravitinoClient client = ... +Group group = + client.addGroup("group1"); +``` + + + + +### Get a group + +You can get a group by its name. + + + + +```shell +curl -X GET -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" http://localhost:8090/api/metalakes/test/groups/group1 +``` + + + + +```java +GravitinoClient client = ... +Group group = + client.getGroup("group1"); +``` + + + + +### Delete a group + +You can delete a group by its name. + + + + +```shell +curl -X DELETE -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" http://localhost:8090/api/metalakes/test/groups/group1 +``` + + + + +```java +GravitinoClient client = ... +boolean deleted = + client.deleteGroup("group1"); +``` + + + + +## Role Operation + +### Create a role + +You can create a role by given properties. + + + + +```shell +curl -X POST -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "role1", + "properties": {"k1", "v1"} + "securableObjects": [ + { + "fullName": "catalog1.schema1.table1", + "type": "TABLE", + "privileges": [ + { + "name": "SELECT_TABLE", + "condition": "ALLOW" + } + ] + } + ] +}' http://localhost:8090/api/metalakes/test/roles +``` + + + + +```java +GravitinoClient client = ... + +SecurableObject securableObject = + SecurableObjects.ofTable( + SecurableObjects.ofSchema( + SecurableObjects.ofCatalog("catalog1", Collections.emptyList()), + "schema1", + Collections.emptyList()), + "table1", + Lists.newArrayList(Privileges.SelectTable.allow())); + +Role role = + client.createRole("role1", ImmutableMap.of("k1", "v1"), Lists.newArrayList(securableObject)); +``` + + + + +### Get a role + +You can get a role by its name. + + + + +```shell +curl -X GET -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d http://localhost:8090/api/metalakes/test/roles/role1 +``` + + + + +```java +GravitinoClient client = ... +Role role = + client.getRole("role1"); +``` + + + + +### Delete a role + +You can delete a role by its name. + + + + +```shell +curl -X DELETE -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" http://localhost:8090/api/metalakes/test/roles/role1 +``` + + + + +```java +GravitinoClient client = ... +boolean deleted = + client.deleteRole("role1"); +``` + + + + +## Permission Operation + +### Grant roles to a user + +You can grant specific roles to a user. + + + + +```shell +curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "roleNames": ["role1"] +}'http://localhost:8090/api/metalakes/test/permissions/users/user1/grant +``` + + + + +```java +GravitinoClient client = ... +User user = client.grantRolesToUser(Lists.newList("role1"), "user1"); +``` + + + + +### Revoke roles from a user + +You can revoke specific roles from a user. + + + + +```shell +curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "roleNames": ["role1"] +}'http://localhost:8090/api/metalakes/test/permissions/users/user1/revoke +``` + + + + +```java +GravitinoClient client = ... +User user = client.revokeRolesFromUser(Lists.newList("role1"), "user1"); +``` + + + + + +### Grant roles to a group + +You can grant specific roles to a group. + + + + +```shell +curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "roleNames": ["role1"] +}'http://localhost:8090/api/metalakes/test/permissions/groups/group1/grant +``` + + + + +```java +GravitinoClient client = ... +Group group = client.grantRolesToGroup(Lists.newList("role1"), "group1"); +``` + + + + +### Revoke roles from a group + +You can revoke specific roles from a group. + + + + +```shell +curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "roleNames": ["role1"] +}'http://localhost:8090/api/metalakes/test/permissions/groups/group1/revoke +``` + + + + +```java +GravitinoClient client = ... +Group group = client.grantRolesToGroup(Lists.newList("role1"), "group1"); +``` + + + + +## Ownership Operation + +### get the owner + +You can get the owner of a metadata object. + + + + +```shell +curl -X GET -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" http://localhost:8090/api/metalakes/test/owners/table/catalog1.schema1.table1 +``` + + + + +```java +GravitinoClient client = ... + +MetadataObject table = + MetadataObjects.of(Lists.newArrayList("catalog1", "schema1", "table1"), MetadataObject.Type.TABLE); + +Owner owner = client.getOwner(table); +``` + + + + +### set the owner + +You can set the owner of a metadata object. + + + + +```shell +curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ +-H "Content-Type: application/json" -d '{ + "name": "user1", + "type": "USER" +}'http://localhost:8090/api/metalakes/test/owners/table/catalog1.schema1.table1 +``` + + + + +```java +GravitinoClient client = ... + +MetadataObject table = + MetadataObjects.of(Lists.newArrayList("catalog1", "schema1", "table1"), MetadataObject.Type.TABLE); + +client.setOwner(table, "user1", "USER"); +``` + + + + +## Example + +You can follow the steps to achieve the authorization of Gravitino. + +![concept_workflow_image](../assets/workflow.png) + +1. Service admin configures the Gravitino server to enable authorization and creates a metalake. + +2. Service admin adds the user `Manager` to the metalake. + +3. Service admin sets the `Manager` as the owner of the metalake. + +4. `Manager` adds the user `Staff`. + +5. `Manager` creates a specific role `catalog_manager` with `CREATE_CATALOG` privilege. + +6. `Manager` grants the role `catalog_manager` to the user `Staff`. + +7. `Staff` creates a Hive type catalog. + +8. `Staff` creates a schema `hive_db` for Hive catalog. + +9. `Staff` creates a table `hive_table` under the schema `hive_db`. + +10. `Staff` creates a MySQL type catalog. + +11. `Staff` creates a schema `mysql_db` for MySQL catalog. + +12. `Staff` creates a table `mysql_table` under the schema `mysql_db`. + +13. `Staff` can use Gravitino connector to query the tables from different catalogs. diff --git a/docs/security/how-to-authenticate.md b/docs/security/how-to-authenticate.md new file mode 100644 index 00000000000..41b62035391 --- /dev/null +++ b/docs/security/how-to-authenticate.md @@ -0,0 +1,165 @@ +--- +title: "How to authenticate" +slug: /security/how-to-authenticate +keyword: security authentication oauth kerberos +license: "This software is licensed under the Apache License version 2." +--- + +## Authentication + +Apache Gravitino supports three kinds of authentication mechanisms: simple, OAuth and Kerberos. +If you don't enable authentication for your client and server explicitly, you will use user `anonymous` to access the server. + +### Simple mode + +If the client sets the simple mode, it will use the value of environment variable `GRAVITINO_USER` as the user. +If the environment variable `GRAVITINO_USER` in the client isn't set, the client uses the user logging in the machine that sends requests. + +For the client side, users can enable `simple` mode by the following code: + +```java +GravitinoClient client = GravitinoClient.builder(uri) + .withMetalake("metalake") + .withSimpleAuth() + .build(); +``` + +### OAuth mode + +Gravitino only supports external OAuth 2.0 servers. To enable OAuth mode, users should follow the steps below. + +- First, users need to guarantee that the external correctly configured OAuth 2.0 server supports Bearer JWT. + +- Then, on the server side, users should set `gravitino.authenticators` as `oauth` and give +`gravitino.authenticator.oauth.defaultSignKey`, `gravitino.authenticator.oauth.serverUri` and +`gravitino.authenticator.oauth.tokenPath` a proper value. +- Next, for the client side, users can enable `OAuth` mode by the following code: + +```java +DefaultOAuth2TokenProvider authDataProvider = DefaultOAuth2TokenProvider.builder() + .withUri("oauth server uri") + .withCredential("yy:xx") + .withPath("oauth/token") + .withScope("test") + .build(); + +GravitinoClient client = GravitinoClient.builder(uri) + .withMetalake("metalake") + .withOAuth(authDataProvider) + .build(); +``` + +### Kerberos mode + +To enable Kerberos mode, users need to guarantee that the server and client have the correct Kerberos configuration. In the server side, users should set `gravitino.authenticators` as `kerberos` and give +`gravitino.authenticator.kerberos.principal` and `gravitino.authenticator.kerberos.keytab` a proper value. For the client side, users can enable `kerberos` mode by the following code: + +```java +// Use keytab to create KerberosTokenProvider +KerberosTokenProvider provider = KerberosTokenProvider.builder() + .withClientPrincipal(clientPrincipal) + .withKeyTabFile(new File(keytabFile)) + .build(); + +// Use ticketCache to create KerberosTokenProvider +KerberosTokenProvider provider = KerberosTokenProvider.builder() + .withClientPrincipal(clientPrincipal) + .build(); + +GravitinoClient client = GravitinoClient.builder(uri) + .withMetalake("metalake") + .withKerberosAuth(provider) + .build(); +``` + +:::info +Now Iceberg REST service doesn't support Kerberos authentication. +The URI must use the hostname of server instead of IP. +::: + +### Server configuration + +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|--------------------------------------------|---------------| +| `gravitino.authenticator` | It is deprecated since Gravitino 0.6.0. Please use `gravitino.authenticators` instead. | `simple` | No | 0.3.0 | +| `gravitino.authenticators` | The authenticators which Gravitino uses, setting as `simple`,`oauth` or `kerberos`. Multiple authenticators are separated by commas. If a request is supported by multiple authenticators simultaneously, the first authenticator will be used by default. | `simple` | No | 0.6.0 | +| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | No | 0.3.0 | +| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | No | 0.3.0 | +| `gravitino.authenticator.oauth.defaultSignKey` | The signing key of JWT when Gravitino uses OAuth as the authenticator. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | No | 0.3.0 | +| `gravitino.authenticator.oauth.serverUri` | The URI of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator.oauth.tokenPath` | The path for token of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator.kerberos.principal` | Indicates the Kerberos principal to be used for HTTP endpoint. Principal should start with `HTTP/`. | (none) | Yes if use `kerberos` as the authenticator | 0.4.0 | +| `gravitino.authenticator.kerberos.keytab` | Location of the keytab file with the credentials for the principal. | (none) | Yes if use `kerberos` as the authenticator | 0.4.0 | + +The signature algorithms that Gravitino supports follows: + +| Name | Description | +|-------|------------------------------------------------| +| HS256 | HMAC using SHA-25A | +| HS384 | HMAC using SHA-384 | +| HS512 | HMAC using SHA-51 | +| RS256 | RSASSA-PKCS-v1_5 using SHA-256 | +| RS384 | RSASSA-PKCS-v1_5 using SHA-384 | +| RS512 | RSASSA-PKCS-v1_5 using SHA-512 | +| ES256 | ECDSA using P-256 and SHA-256 | +| ES384 | ECDSA using P-384 and SHA-384 | +| ES512 | ECDSA using P-521 and SHA-512 | +| PS256 | RSASSA-PSS using SHA-256 and MGF1 with SHA-256 | +| PS384 | RSASSA-PSS using SHA-384 and MGF1 with SHA-384 | +| PS512 | RSASSA-PSS using SHA-512 and MGF1 with SHA-512 | + +### Example + +You can follow the steps to set up an OAuth mode Gravitino server. + +1. Prerequisite + + You need to install the JDK8 and Docker. + +2. Set up an external OAuth 2.0 server + + There is a sample-authorization-server based on [spring-authorization-server](https://github.com/spring-projects/spring-authorization-server/tree/1.0.3). The image has registered client information in the external OAuth 2.0 server + and its clientId is `test`, secret is `test`, scope is `test`. + +```shell + docker run -p 8177:8177 --name sample-auth-server -d datastrato/sample-authorization-server:0.3.0 +``` + +3. Open [the JWK URL of the Authorization server](http://localhost:8177/oauth2/jwks) in the browser and you can get the JWK. + + ![jks_response_image](../assets/jks.png) + +4. Convert the JWK to PEM. You can use the [online tool](https://8gwifi.org/jwkconvertfunctions.jsp#google_vignette) or other tools. + + ![pem_convert_result_image](../assets/pem.png) + +5. Copy the public key and remove the character `\n` and you can get the default signing key of Gravitino server. + +6. You can refer to the [Configurations](../gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. + +```text +gravitino.authenticators = oauth +gravitino.authenticator.oauth.serviceAudience = test +gravitino.authenticator.oauth.defaultSignKey = +gravitino.authenticator.oauth.tokenPath = /oauth2/token +gravitino.authenticator.oauth.serverUri = http://localhost:8177 +``` + +7. Open [the URL of Gravitino server](http://localhost:8090) and login in with clientId `test`, clientSecret `test`, and scope `test`. + + ![oauth_login_image](../assets/oauth.png) + +8. You can also use the curl command to access Gravitino. + +Get access token + +```shell +curl --location --request POST 'http://127.0.0.1:8177/oauth2/token?grant_type=client_credentials&client_id=test&client_secret=test&scope=test' +``` + +Use the access token to request the Gravitino + +```shell +curl -v -X GET -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" -H "Authorization: Bearer " http://localhost:8090/api/version +``` \ No newline at end of file diff --git a/docs/security/how-to-use-cors.md b/docs/security/how-to-use-cors.md new file mode 100644 index 00000000000..243df71f954 --- /dev/null +++ b/docs/security/how-to-use-cors.md @@ -0,0 +1,36 @@ +--- +title: "How to use CORS" +slug: /security/how-to-use-cors +keyword: security cors +license: "This software is licensed under the Apache License version 2." +--- + +## Cross-origin resource filter + +### Server configuration + +| Configuration item | Description | Default value | Required | Since version | +|----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------|----------|---------------| +| `gravitino.server.webserver.enableCorsFilter` | Enable cross-origin resource share filter. | false | No | 0.4.0 | +| `gravitino.server.webserver.allowedOrigins` | A comma separated list of allowed origins to access the resources. The default value is *, which means all origins. | `*` | No | 0.4.0 | +| `gravitino.server.webserver.allowedTimingOrigins` | A comma separated list of allowed origins to time the resource. The default value is the empty string, which means no origins. | `` | No | 0.4.0 | +| `gravitino.server.webserver.allowedMethods` | A comma separated list of allowed HTTP methods used when accessing the resources. The default values are GET, POST, HEAD, and DELETE. | `GET,POST,HEAD,DELETE,PUT` | No | 0.4.0 | +| `gravitino.server.webserver.allowedHeaders` | A comma separated list of allowed HTTP headers specified when accessing the resources. The default value is X-Requested-With,Content-Type,Accept,Origin. If the value is a single *, it accepts all headers. | `X-Requested-With,Content-Type,Accept,Origin` | No | 0.4.0 | +| `gravitino.server.webserver.preflightMaxAgeInSecs` | The number of seconds to cache preflight requests by the client. The default value is 1800 seconds or 30 minutes. | `1800` | No | 0.4.0 | +| `gravitino.server.webserver.allowCredentials` | A boolean indicating if the resource allows requests with credentials. The default value is true. | `true` | No | 0.4.0 | +| `gravitino.server.webserver.exposedHeaders` | A comma separated list of allowed HTTP headers exposed on the client. The default value is the empty list. | `` | No | 0.4.0 | +| `gravitino.server.webserver.chainPreflight` | If true chained preflight requests for normal handling (as an OPTION request). Otherwise, the filter responds to the preflight. The default is true. | `true` | No | 0.4.0 | + +### Apache Iceberg REST service's configuration + +| Configuration item | Description | Default value | Required | Since version | +|------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------|----------|---------------| +| `gravitino.iceberg-rest.enableCorsFilter` | Enable cross-origin resource share filter. | false | No | 0.4.0 | +| `gravitino.iceberg-rest.allowedOrigins` | A comma separated list of allowed origins that access the resources. The default value is *, which means all origins. | `*` | No | 0.4.0 | +| `gravitino.iceberg-rest.allowedTimingOrigins` | A comma separated list of allowed origins that time the resource. The default value is the empty string, which means no origins. | `` | No | 0.4.0 | +| `gravitino.iceberg-rest.allowedMethods` | A comma separated list of allowed HTTP methods used when accessing the resources. The default values are GET, POST, HEAD, and DELETE. | `GET,POST,HEAD,DELETE,PUT` | No | 0.4.0 | +| `gravitino.iceberg-rest.allowedHeaders` | A comma separated list of HTTP allowed headers specified when accessing the resources. The default value is X-Requested-With,Content-Type,Accept,Origin. If the value is a single *, it accepts all headers. | `X-Requested-With,Content-Type,Accept,Origin` | No | 0.4.0 | +| `gravitino.iceberg-rest.preflightMaxAgeInSecs` | The number of seconds to cache preflight requests by the client. The default value is 1800 seconds or 30 minutes. | `1800` | No | 0.4.0 | +| `gravitino.iceberg-rest.allowCredentials` | A boolean indicating if the resource allows requests with credentials. The default value is true. | `true` | No | 0.4.0 | +| `gravitino.iceberg-rest.exposedHeaders` | A comma separated list of allowed HTTP headers exposed on the client. The default value is the empty list. | `` | No | 0.4.0 | +| `gravitino.iceberg-rest.chainPreflight` | If true chained preflight requests for normal handling (as an OPTION request). Otherwise, the filter responds to the preflight. The default is true. | `true` | No | 0.4.0 | diff --git a/docs/security/how-to-use-https.md b/docs/security/how-to-use-https.md new file mode 100644 index 00000000000..e489e743999 --- /dev/null +++ b/docs/security/how-to-use-https.md @@ -0,0 +1,120 @@ +--- +title: "How to use HTTPS" +slug: /security/how-to-use-https +keyword: security HTTPS protocol +license: "This software is licensed under the Apache License version 2." +--- + +## HTTPS + +For users choosing OAuth 2.0 as the authentication method, it is recommended to use HTTPS instead of HTTP. HTTPS encrypts the request headers, offering better protection against smuggling attacks. + +Note that Gravitino cannot simultaneously support both HTTP and HTTPS within a single server instance. If HTTPS is enabled, Gravitino will no longer provide HTTP service. + +Currently, both the Gravitino server and Iceberg REST service can configure and support HTTPS. + +### Apache Gravitino server's configuration + +| Configuration item | Description | Default value | Required | Since version | +|-----------------------------------------------------|--------------------------------------------------------------------|---------------------|---------------------------------------------------|---------------| +| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | +| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | +| `gravitino.server.webserver.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | +| `gravitino.server.webserver.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | +| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | '' (empty string) | No | 0.3.0 | +| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | +| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | + +### Apache Iceberg REST service's configuration + +| Configuration item | Description | Default value | Required | Since version | +|-------------------------------------------------|--------------------------------------------------------------------|-------------------|---------------------------------------------------|---------------| +| `gravitino.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | +| `gravitino.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `9433` | No | 0.3.0 | +| `gravitino.iceberg-rest.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.iceberg-rest.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | +| `gravitino.iceberg-rest.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | +| `gravitino.iceberg-rest.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | '' (empty string) | No | 0.3.0 | +| `gravitino.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | +| `gravitino.iceberg-rest.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.iceberg-rest.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | + +Refer to the "Additional JSSE Standard Names" section of the [Java security guide](https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#jssenames) for the list of protocols related to tlsProtocol. You can find the list of `tlsProtocol` values for Java 8 in this document. + +Refer to the "Additional JSSE Standard Names" section of the [Java security guide](https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#ciphersuites) for the list of protocols related to tlsProtocol. You can find the list of `enableCipherAlgorithms` values for Java 8 in this document. + +### Example + +You can follow the steps to set up an HTTPS server. + +1. Prerequisite + - You need to install the JDK8, wget, and set the environment JAVA_HOME. + - If you want to use the command `curl` to request the Gravitino server, you should install openSSL. +2. Generate the key store + +```shell +cd $JAVA_HOME +bin/keytool -genkeypair -alias localhost \ +-keyalg RSA -keysize 4096 -keypass localhost \ +-sigalg SHA256withRSA \ +-keystore localhost.jks -storetype JKS -storepass localhost \ +-dname "cn=localhost,ou=localhost,o=localhost,l=beijing,st=beijing,c=cn" \ +-validity 36500 +``` + +3. Generate the certificate + +```shell +bin/keytool -export -alias localhost -keystore localhost.jks -file localhost.crt -storepass localhost +``` + +4. Import the certificate + +```shell +bin/keytool -import -alias localhost -keystore jre/lib/security/cacerts -file localhost.crt -storepass changeit -noprompt +``` + +5. You can refer to the [Configurations](../gravitino-server-config.md) and append the configuration to the conf/gravitino.conf. + Configuration doesn't support resolving environment variables, so you should replace `${JAVA_HOME}` with the actual value. + Then, You can start the Gravitino server. + +```text +gravitino.server.webserver.host = localhost +gravitino.server.webserver.enableHttps = true +gravitino.server.webserver.keyStorePath = ${JAVA_HOME}/localhost.jks +gravitino.server.webserver.keyStorePassword = localhost +gravitino.server.webserver.managerPassword = localhost +``` + +6. Request the Gravitino server + +- If you use Java, you can copy the code below to a file named Main.java + +```java +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.client.GravitinoVersion; + +public class Main { + public static void main(String[] args) { + String uri = "https://localhost:8433"; + GravitinoClient client = GravitinoClient.builder(uri).withMetalake("metalake").build(); + GravitinoVersion gravitinoVersion = client.getVersion(); + System.out.println(gravitinoVersion); + } +} +``` + +- If you want to use the command `curl`, you can follow the commands: + +```shell +openssl x509 -inform der -in $JAVA_HOME/localhost.crt -out certificate.pem +curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" https://localhost:8433/api/version +``` diff --git a/docs/security/security.md b/docs/security/security.md new file mode 100644 index 00000000000..62ffa268d02 --- /dev/null +++ b/docs/security/security.md @@ -0,0 +1,22 @@ +--- +title: "Security" +slug: /security/security +keyword: security +license: "This software is licensed under the Apache License version 2." +--- + +## Overview + +Gravitino provides features that ensure the highest levels of security for you. + +## Capabilities + +Gravitino has supported the following security features: + +### [Authentication](how-to-authenticate.md) + +### [HTTPS](how-to-use-https.md) + +### [Access Control](access-control.md) + +### [CORS](how-to-use-cors.md) \ No newline at end of file diff --git a/docs/spark-connector/spark-catalog-iceberg.md b/docs/spark-connector/spark-catalog-iceberg.md index db5fa27c7bf..3bc6166314f 100644 --- a/docs/spark-connector/spark-catalog-iceberg.md +++ b/docs/spark-connector/spark-catalog-iceberg.md @@ -12,7 +12,9 @@ The Apache Gravitino Spark connector offers the capability to read and write Ice #### Support DML and DDL operations: - `CREATE TABLE` - - `Supports basic create table clause including table schema, properties, partition, does not support distribution and sort orders.` + +Doesn't support distribution and sort orders. + - `DROP TABLE` - `ALTER TABLE` - `INSERT INTO&OVERWRITE` @@ -29,7 +31,7 @@ The Apache Gravitino Spark connector offers the capability to read and write Ice - View operations. - Metadata tables, like: - `{iceberg_catalog}.{iceberg_database}.{iceberg_table}.snapshots` -- Other Iceberg extension SQL, like: +- Other Iceberg extension SQLs, like: - `ALTER TABLE prod.db.sample ADD PARTITION FIELD xx` - `ALTER TABLE ... WRITE ORDERED BY` - `ALTER TABLE prod.db.sample CREATE BRANCH branchName` @@ -95,7 +97,7 @@ DESC EXTENDED employee; For more details about `CALL`, please refer to the [Spark Procedures description](https://iceberg.apache.org/docs/1.5.2/spark-procedures/#spark-procedures) in Iceberg official document. -## Apache Iceberg backend-catalog support +## Apache Iceberg catalog backend support - HiveCatalog - JdbcCatalog - RESTCatalog diff --git a/docs/trino-connector/catalog-hive.md b/docs/trino-connector/catalog-hive.md index ba1332f4e14..60b2e247e52 100644 --- a/docs/trino-connector/catalog-hive.md +++ b/docs/trino-connector/catalog-hive.md @@ -1,5 +1,5 @@ --- -title: "Apache Gravitino connector - Hive catalog" +title: "Apache Gravitino Trino connector - Hive catalog" slug: /trino-connector/catalog-hive keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." @@ -50,8 +50,8 @@ CREATE SCHEMA catalog.schema_name ### Create table -The Gravitino connector currently supports basic Hive table creation statements, such as defining fields, -allowing null values, and adding comments. The Gravitino connector does not support `CREATE TABLE AS SELECT`. +The Gravitino Trino connector currently supports basic Hive table creation statements, such as defining fields, +allowing null values, and adding comments. The Gravitino Trino connector does not support `CREATE TABLE AS SELECT`. The following example shows how to create a table in the Hive catalog: @@ -77,7 +77,7 @@ Support for the following alter table operations: ### Select -The Gravitino connector supports most SELECT statements, allowing the execution of queries successfully. +The Gravitino Trino connector supports most SELECT statements, allowing the execution of queries successfully. Currently, it doesn't support certain query optimizations, such as pushdown and pruning functionalities. ## Schema and table properties @@ -290,9 +290,9 @@ DROP TABLE hive_test.database_01.table_01; ## HDFS config and permissions -For basic setups, Gravitino connector configures the HDFS client automatically and does not require any configuration +For basic setups, Gravitino Trino connector configures the HDFS client automatically and does not require any configuration files. -Gravitino connector is not support user to config the `hdfs-site.xml` and `core-site.xml` files to the HDFS client. +Gravitino Trino connector is not support user to config the `hdfs-site.xml` and `core-site.xml` files to the HDFS client. Before running any `Insert` statements for Hive tables in Trino, you must check that the user Trino is using to access HDFS has access to the Hive warehouse directory. @@ -301,4 +301,4 @@ replacing hdfs_user with the appropriate username: ```text -DHADOOP_USER_NAME=hdfs_user -``` \ No newline at end of file +``` diff --git a/docs/trino-connector/catalog-iceberg.md b/docs/trino-connector/catalog-iceberg.md index 19351412bb1..4a06267cafa 100644 --- a/docs/trino-connector/catalog-iceberg.md +++ b/docs/trino-connector/catalog-iceberg.md @@ -1,5 +1,5 @@ --- -title: "Apache Gravitino connector - Iceberg catalog" +title: "Apache Gravitino Trino connector - Iceberg catalog" slug: /trino-connector/catalog-iceberg keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." @@ -32,8 +32,8 @@ CREATE SCHEMA "metalake.catalog".schema_name ### Create table -The Gravitino connector currently supports basic Iceberg table creation statements, such as defining fields, -allowing null values, and adding comments. The Gravitino connector does not support `CREATE TABLE AS SELECT`. +The Gravitino Trino connector currently supports basic Iceberg table creation statements, such as defining fields, +allowing null values, and adding comments. The Gravitino Trino connector does not support `CREATE TABLE AS SELECT`. The following example shows how to create a table in the Iceberg catalog: @@ -57,7 +57,7 @@ Support for the following alter table operations: ## Select -The Gravitino connector supports most SELECT statements, allowing the execution of queries successfully. +The Gravitino Trino connector supports most SELECT statements, allowing the execution of queries successfully. Currently, it doesn't support certain query optimizations, such as pushdown and pruning functionalities. ## Table and Schema properties @@ -236,4 +236,4 @@ replacing hdfs_user with the appropriate username: ```text -DHADOOP_USER_NAME=hdfs_user -``` \ No newline at end of file +``` diff --git a/docs/trino-connector/catalog-mysql.md b/docs/trino-connector/catalog-mysql.md index 035a66860e9..6cbd6ea43da 100644 --- a/docs/trino-connector/catalog-mysql.md +++ b/docs/trino-connector/catalog-mysql.md @@ -1,5 +1,5 @@ --- -title: "Apache Gravitino connector - MySQL catalog" +title: "Apache Gravitino Trino connector - MySQL catalog" slug: /trino-connector/catalog-mysql keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." @@ -16,10 +16,9 @@ To connect to MySQL, you need: ## Create table -At present, the Apache Gravitino connector only supports basic MySQL table creation statements, which involve fields, null allowances, and comments. -However, it does not support advanced features like primary keys, indexes, default values, and auto-increment. +At present, the Apache Gravitino Trino connector only supports basic MySQL table creation statements, which involve fields, null allowances, and comments. However, it does not support advanced features like primary keys, indexes, default values, and auto-increment. -The Gravitino connector does not support `CREATE TABLE AS SELECT`. +The Gravitino Trino connector does not support `CREATE TABLE AS SELECT`. ## Alter table @@ -32,7 +31,7 @@ Support for the following alter table operations: ## Select -The Gravitino connector supports most SELECT statements, allowing the execution of queries successfully. +The Gravitino Trino connector supports most SELECT statements, allowing the execution of queries successfully. Currently, it doesn't support certain query optimizations, such as indexes and pushdowns. ## Table and Schema properties @@ -171,4 +170,4 @@ Drop a table: ```sql DROP TABLE mysql_test.database_01.table_01; -``` \ No newline at end of file +``` diff --git a/docs/trino-connector/catalog-postgresql.md b/docs/trino-connector/catalog-postgresql.md index 6ae6d7fe346..9ce2876ff56 100644 --- a/docs/trino-connector/catalog-postgresql.md +++ b/docs/trino-connector/catalog-postgresql.md @@ -1,5 +1,5 @@ --- -title: "Apache Gravitino connector - PostgreSQL catalog" +title: "Apache Gravitino Trino connector - PostgreSQL catalog" slug: /trino-connector/catalog-postgresql keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." @@ -16,14 +16,13 @@ To connect to PostgreSQL, you need: ## Create table -At present, the Apache Gravitino connector only supports basic PostgreSQL table creation statements, which involve fields, null allowances, and comments. -However, it does not support advanced features like primary keys, indexes, default values, and auto-increment. +At present, the Apache Gravitino Trino connector only supports basic PostgreSQL table creation statements, which involve fields, null allowances, and comments. However, it does not support advanced features like primary keys, indexes, default values, and auto-increment. -The Gravitino connector does not support `CREATE TABLE AS SELECT`. +The Gravitino Trino connector does not support `CREATE TABLE AS SELECT`. ## Alter table -Gravitino connector supports the following alter table operations: +Gravitino Trino connector supports the following alter table operations: - Rename table - Add a column - Drop a column @@ -33,7 +32,7 @@ Gravitino connector supports the following alter table operations: ## Select -The Gravitino connector supports most SELECT statements, allowing the execution of queries successfully. +The Gravitino Trino connector supports most SELECT statements, allowing the execution of queries successfully. Currently, it doesn't support certain query optimizations, such as indexes and pushdowns. ## Table and Schema properties @@ -171,4 +170,4 @@ Drop a table: ```sql DROP TABLE postgresql_test.database_01.table_01; -``` \ No newline at end of file +``` diff --git a/docs/trino-connector/configuration.md b/docs/trino-connector/configuration.md index 25d5d48712e..ba2ac18f5f8 100644 --- a/docs/trino-connector/configuration.md +++ b/docs/trino-connector/configuration.md @@ -1,5 +1,5 @@ --- -title: "Apache Gravitino connector Configuration" +title: "Apache Gravitino Trino connector Configuration" slug: /trino-connector/configuration keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." diff --git a/docs/trino-connector/development.md b/docs/trino-connector/development.md index 469d591c2e8..509a44a0f58 100644 --- a/docs/trino-connector/development.md +++ b/docs/trino-connector/development.md @@ -1,15 +1,15 @@ --- -title: "Apache Gravitino connector development" +title: "Apache Gravitino Trino connector development" slug: /trino-connector/development keyword: gravitino connector development license: "This software is licensed under the Apache License version 2." --- -This document is to guide users through the development of the Apache Gravitino connector for Trino locally. +This document is to guide users through the development of the Apache Gravitino Trino connector for Trino locally. ## Prerequisites -Before you start developing the Gravitino trino connector, you need to have the following prerequisites: +Before you start developing the Gravitino Trino connector, you need to have the following prerequisites: 1. You need to start the Gravitino server locally, for more information, please refer to the [start Gravitino server](../how-to-install.md) 2. Create a catalog in the Gravitino server, for more information, please refer to the [Gravitino metadata management](../manage-relational-metadata-using-gravitino.md). Assuming we have just created a MySQL catalog using the following command: @@ -32,15 +32,14 @@ Please change the above `localhost`, `port` and the names of metalake and catalo ## Development environment -To develop the Gravitino connector locally, you need to do the following steps: +To develop the Gravitino Trino connector locally, you need to do the following steps: ### IDEA 1. Clone the Trino repository from the [GitHub](https://github.com/trinodb/trino) repository. The released version Trino-435 is the least version that Gravitino supports. 2. Open the Trino project in your IDEA. -3. Create a new module for the Gravitino connector in the Trino project as the following picture (we will use the name `trino-gravitino` as the module name in the following steps). ![trino-gravitino](../assets/trino/create-gravitino-connector.jpg) -4. Add a soft link to the Gravitino trino connector module in the Trino project. Assuming the src java main directory of the Gravitino trino connector in project Gravitino is `gravitino/path/to/gravitino-trino-connector/src/main/java`, -and the src java main directory of trino-gravitino in the Trino project is `trino/path/to/trino-gravitino/src/main/java`, you can use the following command to create a soft link: +3. Create a new module for the Gravitino Trino connector in the Trino project as the following picture (we will use the name `trino-gravitino` as the module name in the following steps). ![trino-gravitino](../assets/trino/create-gravitino-trino-connector.jpg) +4. Add a soft link to the Gravitino Trino connector module in the Trino project. Assuming the src java main directory of the Gravitino Trino connector in project Gravitino is `gravitino/path/to/gravitino-trino-connector/src/main/java`, and the src java main directory of trino-gravitino in the Trino project is `trino/path/to/trino-gravitino/src/main/java`, you can use the following command to create a soft link: ```shell ln -s gravitino/path/to/trino-connector/src/main/java trino/path/to/trino-gravitino/src/main/java @@ -162,7 +161,7 @@ then you can see the `gravitino-trino-connecor` source files and directories in If a compile error occurs due to `The following artifacts could not be resolved: org.apache.gravitino:xxx:jar`, which can be resolved by executing `./gradlew publishToMavenLocal` in gravitino beforehand. ::: -7. Set up the configuration for the Gravitino connector in the Trino project. You can do as the following picture shows: +7. Set up the configuration for the Gravitino Trino connector in the Trino project. You can do as the following picture shows: ![](../assets/trino/add-config.jpg) The corresponding configuration files are here: @@ -222,7 +221,7 @@ plugin.bundles=\ node-scheduler.include-coordinator=true -# Note: The Gravitino connector olny supports with The dynamic catalog manager +# Note: The Gravitino Trino connector olny supports with The dynamic catalog manager catalog.management=dynamic ``` @@ -232,7 +231,7 @@ Remove the file `/etc/catalogs/xxx.properties` if the corresponding `plugin/trin 8. Start the Trino server and connect to the Gravitino server. ![](../assets/trino/start-trino.jpg) -9. If `DevelopmentServer` has started successfully, you can connect to the Trino server using the `trino-cli` and run the following command to see if the Gravitino connector is available: +9. If `DevelopmentServer` has started successfully, you can connect to the Trino server using the `trino-cli` and run the following command to see if the Gravitino Trino connector is available: ```shell java -jar trino-cli-429-executable.jar --server localhost:8180 ``` @@ -240,5 +239,5 @@ java -jar trino-cli-429-executable.jar --server localhost:8180 The `trino-cli-429-executable.jar` is the Trino CLI jar file, you can download it from the [Trino release page](https://trino.io/docs/current/client/cli.html). **Users can use the version of the Trino CLI jar file according to the version of the Trino server.** ::: -10. If nothing goes wrong, you can start developing the Gravitino connector in the Gravitino project and debug it in the Trino project. +10. If nothing goes wrong, you can start developing the Gravitino Trino connector in the Gravitino project and debug it in the Trino project. ![](../assets/trino/show-catalogs.jpg) diff --git a/docs/trino-connector/index.md b/docs/trino-connector/index.md index 8e44833ca36..ecd88673ecd 100644 --- a/docs/trino-connector/index.md +++ b/docs/trino-connector/index.md @@ -1,11 +1,11 @@ --- -title: "Apache Gravitino connector index" +title: "Apache Gravitino Trino connector index" slug: /trino-connector/index keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." --- -Apache Gravitino connector index: +Apache Gravitino Trino connector index: - [Trino Support](trino-connector.md) - [Requirements](requirements.md) diff --git a/docs/trino-connector/installation.md b/docs/trino-connector/installation.md index 0b4f0db6df4..d399007bb4e 100644 --- a/docs/trino-connector/installation.md +++ b/docs/trino-connector/installation.md @@ -1,19 +1,19 @@ --- -title: "Apache Gravitino connector installation" +title: "Apache Gravitino Trino connector installation" slug: /trino-connector/installation keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." --- -To install the Apache Gravitino connector, you should first deploy the Trino environment, and then install the Gravitino connector plugin into Trino. +To install the Apache Gravitino Trino connector, you should first deploy the Trino environment, and then install the Gravitino Trino connector plugin into Trino. Please refer to the [Deploying Trino documentation](https://trino.io/docs/current/installation/deployment.html) and do the following steps: -1. [Download](https://github.com/apache/gravitino/releases) the Gravitino connector tarball and unpack it. +1. [Download](https://github.com/apache/gravitino/releases) the Gravitino Trino connector tarball and unpack it. The tarball contains a single top-level directory `gravitino-trino-connector-`, which called the connector directory. 2. Copy the connector directory to the Trino's plugin directory. Normally, the directory location is `Trino-server-/plugin`, and the directory contains other catalogs used by Trino. -3. Add Trino JVM arguments `-Dlog4j.configurationFile=file:////etc/trino/log4j2.properties` to enable logging for the Gravitino connector. +3. Add Trino JVM arguments `-Dlog4j.configurationFile=file:////etc/trino/log4j2.properties` to enable logging for the Gravitino Trino connector. 4. Update Trino coordinator configuration. You need to set `catalog.management=dynamic`, The config location is `Trino-server-/etc/config.properteis`, and the contents like: @@ -26,13 +26,13 @@ discovery.uri=http://0.0.0.0:8080 ``` Alternatively, -you can build the Gravitino connector package from the sources +you can build the Gravitino Trino connector package from the sources and obtain the `gravitino-trino-connector-.tar.gz` file in the `$PROJECT/distribution` directory. Please refer to the [Gravitino Development documentation](../how-to-build.md) ## Example -You can install the Gravitino connector in Trino office docker images step by step. +You can install the Gravitino Trino connector in Trino office docker images step by step. ### Running the container @@ -46,9 +46,9 @@ docker run --name trino-gravitino -d -p 8080:8080 trinodb/trino:435 Run `docker ps` to check whether the container is running. -### Installing the Apache Gravitino connector +### Installing the Apache Gravitino Trino connector -Download the Gravitino connector tarball and unpack it. +Download the Gravitino Trino connector tarball and unpack it. ```shell cd /tmp @@ -71,7 +71,7 @@ docker exec -it trino-gravitino /bin/bash cd /lib/trino/plugin ``` -Now you can see the Gravitino connector directory in the plugin directory. +Now you can see the Gravitino Trino connector directory in the plugin directory. ### Configuring the Trino @@ -86,11 +86,11 @@ discovery.uri=http://localhost:8080 catalog.management=dynamic ``` -### Configuring the Apache Gravitino connector +### Configuring the Apache Gravitino Trino connector Assuming you have now started the Gravitino server on the host `gravitino-server-host` and already created a metalake named `test`, if those have not been prepared, please refer to the [Gravitino getting started](../getting-started.md). -To configure Gravitino connector correctly, you need to put the following configurations to the Trino configuration file `/etc/trino/catalog/gravitino.properties`. +To configure Gravitino Trino connector correctly, you need to put the following configurations to the Trino configuration file `/etc/trino/catalog/gravitino.properties`. ```text connector.name=gravitino @@ -98,11 +98,11 @@ gravitino.uri=http://gravitino-server-host:8090 gravitino.metalake=test ``` -- The `gravitino.name` defines which Gravitino connector is used. It must be `gravitino`. +- The `gravitino.name` defines which Gravitino Trino connector is used. It must be `gravitino`. - The `gravitino.metalake` defines which metalake are used. It should exist in the Gravitino server. - The `gravitino.uri` defines the connection information about Gravitino server. Make sure your container can access the Gravitino server. -Full configurations for Apache Gravitino connector can be seen [here](configuration.md) +Full configurations for Apache Gravitino Trino connector can be seen [here](configuration.md) If you haven't created the metalake named `test`, you can use the following command to create it. @@ -110,13 +110,13 @@ If you haven't created the metalake named `test`, you can use the following comm curl -X POST -H "Content-Type: application/json" -d '{"name":"test","comment":"comment","properties":{}}' http://gravitino-server-host:8090/api/metalakes ``` -And then restart the Trino container to load the Gravitino connector. +And then restart the Trino container to load the Gravitino Trino connector. ```shell docker restart trino-gravitino ``` -### Verifying the Apache Gravitino connector +### Verifying the Apache Gravitino Trino connector Use the Trino CLI to connect to the Trino container and run a query. @@ -133,7 +133,7 @@ tpch system ``` -You can see the `gravitino` catalog in the result set. This signifies the successful installation of the Gravitino connector. +You can see the `gravitino` catalog in the result set. This signifies the successful installation of the Gravitino Trino connector. Assuming you have created a catalog named `test.jdbc-mysql` in the Gravitino server, or please refer to [Create a Catalog](../manage-relational-metadata-using-gravitino.md#create-a-catalog). Then you can use the Trino CLI to connect to the Trino container and run a query like this. diff --git a/docs/trino-connector/requirements.md b/docs/trino-connector/requirements.md index 903d2f9df1d..1723e6f9a11 100644 --- a/docs/trino-connector/requirements.md +++ b/docs/trino-connector/requirements.md @@ -1,11 +1,11 @@ --- -title: "Apache Gravitino connector requirements" +title: "Apache Gravitino Trino connector requirements" slug: /trino-connector/requirements keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." --- -To install and deploy the Apache Gravitino connector, The following environmental setup is necessary: +To install and deploy the Apache Gravitino Trino connector, The following environmental setup is necessary: - Trino server version should be between Trino-server-435 and Trino-server-439. Other versions of Trino have not undergone thorough testing. diff --git a/docs/trino-connector/sql-support.md b/docs/trino-connector/sql-support.md index 934e989a0c1..6603abf820b 100644 --- a/docs/trino-connector/sql-support.md +++ b/docs/trino-connector/sql-support.md @@ -1,5 +1,5 @@ --- -title: "Apache Gravitino connector SQL support" +title: "Apache Gravitino Trino connector SQL support" slug: /trino-connector/sql-support keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." @@ -45,4 +45,4 @@ The connector provides read access and write access to data and metadata stored - [COMMIT](https://trino.io/docs/current/sql/commit.html) - [ROLLBACK](https://trino.io/docs/current/sql/rollback.html) -For more information, please refer to Trino [SQL statements support](https://trino.io/docs/current/language/sql-support.html#sql-globally-available) \ No newline at end of file +For more information, please refer to Trino [SQL statements support](https://trino.io/docs/current/language/sql-support.html#sql-globally-available) diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md index 306b2dcc2da..296514b5889 100644 --- a/docs/trino-connector/supported-catalog.md +++ b/docs/trino-connector/supported-catalog.md @@ -1,11 +1,11 @@ --- -title: "Apache Gravitino supported Catalogs" +title: "Apache Gravitino Trino connector supported Catalogs" slug: /trino-connector/supported-catalog keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." --- -The catalogs currently supported by the Apache Gravitino connector are as follows: +The catalogs currently supported by the Apache Gravitino Trino connector are as follows: - [Hive](catalog-hive.md) - [Iceberg](catalog-iceberg.md) @@ -14,8 +14,8 @@ The catalogs currently supported by the Apache Gravitino connector are as follow ## Create catalog -Users can create catalogs through the Gravitino connector and then load them into Trino. -The Gravitino connector provides the following stored procedures to create, delete, and alter catalogs. +Users can create catalogs through the Gravitino Trino connector and then load them into Trino. +The Gravitino Trino connector provides the following stored procedures to create, delete, and alter catalogs. User can also use the system table `catalog` to describe all the catalogs. Create catalog: @@ -136,9 +136,9 @@ call gravitino.system.create_catalog( ); ``` -A prefix with `trino.bypass.` in the configuration key is used to indicate Gravitino connector to pass the Trino connector configuration to the Gravitino catalog in the Trino runtime. +A prefix with `trino.bypass.` in the configuration key is used to indicate Gravitino Trino connector to pass the Trino connector configuration to the Gravitino catalog in the Trino runtime. -More trino connector configurations can refer to: +More Trino connector configurations can refer to: - [Hive catalog](https://trino.io/docs/current/connector/hive.html#hive-general-configuration-properties) - [Iceberg catalog](https://trino.io/docs/current/connector/iceberg.html#general-configuration) - [MySQL catalog](https://trino.io/docs/current/connector/mysql.html#general-configuration-properties) @@ -146,7 +146,7 @@ More trino connector configurations can refer to: ## Data type mapping between Trino and Apache Gravitino -Gravitino connector supports the following data type conversions between Trino and Gravitino currently. Depending on the detailed catalog, Gravitino may not support some data types conversion for this specific catalog, for example, +Gravitino Trino connector supports the following data type conversions between Trino and Gravitino currently. Depending on the detailed catalog, Gravitino may not support some data types conversion for this specific catalog, for example, Hive does not support `TIME` data type. | Gravitino Type | Trino Type | diff --git a/docs/trino-connector/trino-connector.md b/docs/trino-connector/trino-connector.md index f1167fe9a49..37cb67b1d0a 100644 --- a/docs/trino-connector/trino-connector.md +++ b/docs/trino-connector/trino-connector.md @@ -1,18 +1,18 @@ --- -title: "Apache Gravitino connector" +title: "Apache Gravitino Trino connector" slug: /trino-connector/trino-connector keyword: gravitino connector trino license: "This software is licensed under the Apache License version 2." --- -Trino can manage and access data using the Trino connector provided by `Apache Gravitino`, commonly referred to as the `Gravitino connector`. -After configuring the Gravitino connector in Trino, Trino can automatically load catalog metadata from Gravitino, allowing users to directly access these catalogs in Trino. +Trino can manage and access data using the Trino connector provided by `Apache Gravitino`, commonly referred to as the `Gravitino Trino connector`. +After configuring the Gravitino Trino connector in Trino, Trino can automatically load catalog metadata from Gravitino, allowing users to directly access these catalogs in Trino. Once integrated with Gravitino, Trino can operate on all Gravitino data without requiring additional configuration. -The Gravitino connector uses the [Trino dynamic catalog managed mechanism](https://trino.io/docs/current/admin/properties-catalog.html) to load catalogs. -When the Gravitino connector retrieves catalogs from the Gravitino server, it generates a `CREATE CATAGLOG` statement and executes +The Gravitino Trino connector uses the [Trino dynamic catalog managed mechanism](https://trino.io/docs/current/admin/properties-catalog.html) to load catalogs. +When the Gravitino Trino connector retrieves catalogs from the Gravitino server, it generates a `CREATE CATAGLOG` statement and executes the statement on the current Trino server to register the catalogs with Trino -:::node +:::note Once metadata such as catalogs are changed in Gravitino, Trino can update itself through Gravitino, this process usually takes about 3~10 seconds. ::: diff --git a/docs/webui.md b/docs/webui.md index 94ebefa1525..3a174234062 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -10,15 +10,15 @@ license: 'This software is licensed under the Apache License version 2.' This document primarily outlines how users can manage metadata within Apache Gravitino using the web UI, the graphical interface is accessible through a web browser as an alternative to writing code or using the REST interface. -Currently, you can integrate [OAuth settings](./security.md) to view, add, modify, and delete metalakes, create catalogs, and view catalogs, schemas, and tables, among other functions. +Currently, you can integrate [OAuth settings](security/security.md) to view, add, modify, and delete metalakes, create catalogs, and view catalogs, schemas, and tables, among other functions. [Build](./how-to-build.md#quick-start) and [deploy](./getting-started.md#getting-started-locally) the Gravitino Web UI and open it in a browser at `http://:`, by default is [http://localhost:8090](http://localhost:8090). ## Initial page -The web UI homepage displayed in Gravitino depends on the configuration parameter for OAuth mode, see the details in [Security](./security.md). +The web UI homepage displayed in Gravitino depends on the configuration parameter for OAuth mode, see the details in [Security](security/security.md). -Set parameter for `gravitino.authenticator`, [`simple`](#simple-mode) or [`oauth`](#oauth-mode). Simple mode is the default authentication option. +Set parameter for `gravitino.authenticators`, [`simple`](#simple-mode) or [`oauth`](#oauth-mode). Simple mode is the default authentication option. If multiple authenticators are set, the first one is taken by default. :::tip After changing the configuration, make sure to restart the Gravitino server. @@ -29,10 +29,10 @@ After changing the configuration, make sure to restart the Gravitino server. ### Simple mode ```text -gravitino.authenticator = simple +gravitino.authenticators = simple ``` -Set the configuration parameter `gravitino.authenticator` to `simple`, and the web UI displays the homepage (Metalakes). +Set the configuration parameter `gravitino.authenticators` to `simple`, and the web UI displays the homepage (Metalakes). ![webui-metalakes-simple](./assets/webui/metalakes-simple.png) @@ -43,10 +43,10 @@ The main content displays the existing metalake list. ### Oauth mode ```text -gravitino.authenticator = oauth +gravitino.authenticators = oauth ``` -Set the configuration parameter `gravitino.authenticator` to `oauth`, and the web UI displays the login page. +Set the configuration parameter `gravitino.authenticators` to `oauth`, and the web UI displays the login page. :::caution If both `OAuth` and `HTTPS` are set, due to the different security permission rules of various browsers, to avoid cross-domain errors, @@ -57,7 +57,7 @@ Such as Safari need to enable the developer menu, and select `Disable Cross-Orig ![webui-login-with-oauth](./assets/webui/login-with-oauth.png) -1. Enter the values corresponding to your specific configuration. For detailed instructions, please refer to [Security](./security.md). +1. Enter the values corresponding to your specific configuration. For detailed instructions, please refer to [Security](security/security.md). 2. Clicking on the `LOGIN` button takes you to the homepage. diff --git a/flink-connector/build.gradle.kts b/flink-connector/build.gradle.kts index a21dea4a1ba..169b5f80988 100644 --- a/flink-connector/build.gradle.kts +++ b/flink-connector/build.gradle.kts @@ -34,7 +34,7 @@ val flinkVersion: String = libs.versions.flink.get() // https://issues.apache.org/jira/browse/FLINK-20845, // https://issues.apache.org/jira/browse/FLINK-13414. val scalaVersion: String = "2.12" -val artifactName = "gravitino-${project.name}-$scalaVersion" +val artifactName = "gravitino-${project.name}_$scalaVersion" dependencies { implementation(project(":api")) diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java index 8bf9b86d019..b75fc88d672 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java @@ -158,6 +158,14 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .stringConf() .create(); + public static final ConfigEntry ICEBERG_REST_CATALOG_PROVIDER = + new ConfigBuilder(IcebergConstants.ICEBERG_REST_CATALOG_PROVIDER) + .doc( + "Catalog provider class name, you can develop a class that implements `IcebergTableOpsProvider` and add the corresponding jar file to the Iceberg REST service classpath directory.") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .createWithDefault("config-based-provider"); + public String getJdbcDriver() { return get(JDBC_DRIVER); } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/ConfigBasedIcebergTableOpsProvider.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/ConfigBasedIcebergTableOpsProvider.java new file mode 100644 index 00000000000..070e67ce1cb --- /dev/null +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/ConfigBasedIcebergTableOpsProvider.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.common.ops; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.utils.MapUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This provider use configs to support multiple catalogs. + * + *

For example, there are two different catalogs: jdbc_proxy, hive_proxy The config is like: + * + *

gravitino.iceberg-rest.catalog.jdbc_proxy.catalog-backend = jdbc + * gravitino.iceberg-rest.catalog.jdbc_proxy.uri = jdbc:mysql://{host}:{port}/{db} ... + * gravitino.iceberg-rest.catalog.hive_proxy.catalog-backend = hive + * gravitino.iceberg-rest.catalog.hive_proxy.uri = thrift://{host}:{port} ... + */ +public class ConfigBasedIcebergTableOpsProvider implements IcebergTableOpsProvider { + public static final Logger LOG = + LoggerFactory.getLogger(ConfigBasedIcebergTableOpsProvider.class); + + public static final String CONFIG_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME = "config-based-provider"; + + @VisibleForTesting Map catalogConfigs; + + @Override + public void initialize(Map properties) { + this.catalogConfigs = + properties.keySet().stream() + .map(this::getCatalogName) + .filter(Optional::isPresent) + .map(Optional::get) + .distinct() + .collect( + Collectors.toMap( + catalogName -> catalogName, + catalogName -> + new IcebergConfig( + MapUtils.getPrefixMap( + properties, String.format("catalog.%s.", catalogName))))); + this.catalogConfigs.put( + IcebergConstants.GRAVITINO_DEFAULT_CATALOG, new IcebergConfig(properties)); + } + + @Override + public IcebergTableOps getIcebergTableOps(String catalogName) { + IcebergConfig icebergConfig = this.catalogConfigs.get(catalogName); + if (icebergConfig == null) { + String errorMsg = String.format("%s can not match any catalog", catalogName); + LOG.warn(errorMsg); + throw new RuntimeException(errorMsg); + } + return new IcebergTableOps(icebergConfig); + } + + private Optional getCatalogName(String catalogConfigKey) { + if (!catalogConfigKey.startsWith("catalog.")) { + return Optional.empty(); + } + // The catalogConfigKey's format is catalog.. + if (catalogConfigKey.split("\\.").length < 3) { + LOG.warn("{} format is illegal", catalogConfigKey); + return Optional.empty(); + } + return Optional.of(catalogConfigKey.split("\\.")[1]); + } +} diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java index 56ca6e505aa..0720c0abba8 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOps.java @@ -36,6 +36,8 @@ import org.apache.gravitino.iceberg.common.utils.IcebergCatalogUtil; import org.apache.gravitino.utils.IsolatedClassLoader; import org.apache.gravitino.utils.MapUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; @@ -66,6 +68,7 @@ public class IcebergTableOps implements AutoCloseable { private final IcebergCatalogBackend catalogBackend; private String catalogUri = null; private Map catalogConfigToClients; + private Map catalogPropertiesMap; private static final Set catalogPropertiesToClientKeys = ImmutableSet.of( IcebergConstants.IO_IMPL, @@ -91,6 +94,8 @@ public IcebergTableOps(IcebergConfig icebergConfig) { MapUtils.getFilteredMap( icebergConfig.getIcebergCatalogProperties(), key -> catalogPropertiesToClientKeys.contains(key)); + + this.catalogPropertiesMap = icebergConfig.getIcebergCatalogProperties(); } public IcebergTableOps() { @@ -139,6 +144,19 @@ public LoadTableResponse registerTable(Namespace namespace, RegisterTableRequest return CatalogHandlers.registerTable(catalog, namespace, request); } + /** + * Reload hadoop configuration, this is useful when the hadoop configuration UserGroupInformation + * is shared by multiple threads. UserGroupInformation#authenticationMethod was first initialized + * in KerberosClient, however, when switching to iceberg-rest thead, + * UserGroupInformation#authenticationMethod will be reset to the default value; we need to + * reinitialize it again. + */ + public void reloadHadoopConf() { + Configuration configuration = new Configuration(); + this.catalogPropertiesMap.forEach(configuration::set); + UserGroupInformation.setConfiguration(configuration); + } + public LoadTableResponse createTable(Namespace namespace, CreateTableRequest request) { request.validate(); if (request.stageCreate()) { diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsManager.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsManager.java new file mode 100644 index 00000000000..57792b89f12 --- /dev/null +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsManager.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.common.ops; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IcebergTableOpsManager implements AutoCloseable { + public static final Logger LOG = LoggerFactory.getLogger(IcebergTableOpsManager.class); + + private static final ImmutableMap ICEBERG_TABLE_OPS_PROVIDER_NAMES = + ImmutableMap.of( + ConfigBasedIcebergTableOpsProvider.CONFIG_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME, + ConfigBasedIcebergTableOpsProvider.class.getCanonicalName()); + + private final Cache icebergTableOpsCache; + + private final IcebergTableOpsProvider provider; + + public IcebergTableOpsManager(Map properties) { + this.icebergTableOpsCache = Caffeine.newBuilder().build(); + this.provider = createProvider(properties); + this.provider.initialize(properties); + } + + /** + * @param rawPrefix The path parameter is passed by a Jetty handler. The pattern is matching + * ([^/]*\/), end with / + * @return the instance of IcebergTableOps. + */ + public IcebergTableOps getOps(String rawPrefix) { + String catalogName = getCatalogName(rawPrefix); + IcebergTableOps tableOps = + icebergTableOpsCache.get(catalogName, k -> provider.getIcebergTableOps(catalogName)); + // Reload conf to reset UserGroupInformation or icebergTableOps will always use + // Simple auth. + tableOps.reloadHadoopConf(); + return tableOps; + } + + private String getCatalogName(String rawPrefix) { + String prefix = shelling(rawPrefix); + Preconditions.checkArgument( + !IcebergConstants.GRAVITINO_DEFAULT_CATALOG.equals(prefix), + String.format("%s is conflict with reserved key, please replace it", prefix)); + if (StringUtils.isBlank(prefix)) { + return IcebergConstants.GRAVITINO_DEFAULT_CATALOG; + } + return prefix; + } + + private IcebergTableOpsProvider createProvider(Map properties) { + String providerName = + (new IcebergConfig(properties)).get(IcebergConfig.ICEBERG_REST_CATALOG_PROVIDER); + String className = ICEBERG_TABLE_OPS_PROVIDER_NAMES.getOrDefault(providerName, providerName); + LOG.info("Load Iceberg catalog provider: {}.", className); + try { + Class providerClz = Class.forName(className); + return (IcebergTableOpsProvider) providerClz.getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String shelling(String rawPrefix) { + if (StringUtils.isBlank(rawPrefix)) { + return rawPrefix; + } else { + // rawPrefix is a string matching ([^/]*/) which end with / + Preconditions.checkArgument( + rawPrefix.endsWith("/"), String.format("rawPrefix %s format is illegal", rawPrefix)); + return rawPrefix.substring(0, rawPrefix.length() - 1); + } + } + + @Override + public void close() throws Exception { + icebergTableOpsCache.invalidateAll(); + } +} diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsProvider.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsProvider.java new file mode 100644 index 00000000000..cda5ac20ae8 --- /dev/null +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergTableOpsProvider.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.common.ops; + +import java.util.Map; + +/** + * IcebergTableOpsProvider is an interface defining how Iceberg REST catalog server gets Iceberg + * catalogs. + */ +public interface IcebergTableOpsProvider { + + /** + * @param properties The parameters for creating Provider which from configurations whose prefix + * is 'gravitino.iceberg-rest.' + */ + void initialize(Map properties); + + /** + * @param catalogName a param send by clients. + * @return the instance of IcebergTableOps. + */ + IcebergTableOps getIcebergTableOps(String catalogName); +} diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java index 8f171bea65c..f6e8b4e36b6 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java @@ -99,8 +99,11 @@ private static KerberosClient initKerberosAndReturnClient( Map properties, Configuration conf) { try { KerberosClient kerberosClient = new KerberosClient(properties, conf); - File keytabFile = - kerberosClient.saveKeyTabFileFromUri(Long.valueOf(properties.get("catalog_uuid"))); + + // For Iceberg rest server, we haven't set the catalog_uuid, so we set it to 0 as there is + // only one catalog in the rest server, so it's okay to set it to 0. + String catalogUUID = properties.getOrDefault("catalog_uuid", "0"); + File keytabFile = kerberosClient.saveKeyTabFileFromUri(Long.valueOf(catalogUUID)); kerberosClient.login(keytabFile.getAbsolutePath()); return kerberosClient; } catch (IOException e) { diff --git a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/ops/TestConfigBasedIcebergTableOpsProvider.java b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/ops/TestConfigBasedIcebergTableOpsProvider.java new file mode 100644 index 00000000000..c3c8c3fe8c5 --- /dev/null +++ b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/ops/TestConfigBasedIcebergTableOpsProvider.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.common.ops; + +import com.google.common.collect.Maps; +import java.util.Map; +import java.util.UUID; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.iceberg.hive.HiveCatalog; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.jdbc.JdbcCatalog; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class TestConfigBasedIcebergTableOpsProvider { + private static final String STORE_PATH = + "/tmp/gravitino_test_iceberg_jdbc_backend_" + UUID.randomUUID().toString().replace("-", ""); + + @Test + public void testValidIcebergTableOps() { + String hiveCatalogName = "hive_backend"; + String jdbcCatalogName = "jdbc_backend"; + String defaultCatalogName = IcebergConstants.GRAVITINO_DEFAULT_CATALOG; + + Map config = Maps.newHashMap(); + // hive backend catalog + config.put("catalog.hive_backend.catalog-backend-name", hiveCatalogName); + config.put("catalog.hive_backend.catalog-backend", "hive"); + config.put("catalog.hive_backend.uri", "thrift://127.0.0.1:9083"); + config.put("catalog.hive_backend.warehouse", "/tmp/usr/hive/warehouse"); + // jdbc backend catalog + config.put("catalog.jdbc_backend.catalog-backend-name", jdbcCatalogName); + config.put("catalog.jdbc_backend.catalog-backend", "jdbc"); + config.put( + "catalog.jdbc_backend.uri", + String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", STORE_PATH)); + config.put("catalog.jdbc_backend.warehouse", "/tmp/usr/jdbc/warehouse"); + config.put("catalog.jdbc_backend.jdbc.password", "gravitino"); + config.put("catalog.jdbc_backend.jdbc.user", "gravitino"); + config.put("catalog.jdbc_backend.jdbc-driver", "org.h2.Driver"); + config.put("catalog.jdbc_backend.jdbc-initialize", "true"); + // default catalog + config.put("catalog-backend-name", defaultCatalogName); + config.put("catalog-backend", "memory"); + config.put("warehouse", "/tmp/"); + + ConfigBasedIcebergTableOpsProvider provider = new ConfigBasedIcebergTableOpsProvider(); + provider.initialize(config); + + IcebergConfig hiveIcebergConfig = provider.catalogConfigs.get(hiveCatalogName); + IcebergConfig jdbcIcebergConfig = provider.catalogConfigs.get(jdbcCatalogName); + IcebergConfig defaultIcebergConfig = provider.catalogConfigs.get(defaultCatalogName); + IcebergTableOps hiveOps = provider.getIcebergTableOps(hiveCatalogName); + IcebergTableOps jdbcOps = provider.getIcebergTableOps(jdbcCatalogName); + IcebergTableOps defaultOps = provider.getIcebergTableOps(defaultCatalogName); + + Assertions.assertEquals( + hiveCatalogName, hiveIcebergConfig.get(IcebergConfig.CATALOG_BACKEND_NAME)); + Assertions.assertEquals("hive", hiveIcebergConfig.get(IcebergConfig.CATALOG_BACKEND)); + Assertions.assertEquals( + "thrift://127.0.0.1:9083", hiveIcebergConfig.get(IcebergConfig.CATALOG_URI)); + Assertions.assertEquals( + "/tmp/usr/hive/warehouse", hiveIcebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)); + + Assertions.assertEquals( + jdbcCatalogName, jdbcIcebergConfig.get(IcebergConfig.CATALOG_BACKEND_NAME)); + Assertions.assertEquals("jdbc", jdbcIcebergConfig.get(IcebergConfig.CATALOG_BACKEND)); + Assertions.assertEquals( + "/tmp/usr/jdbc/warehouse", jdbcIcebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)); + Assertions.assertEquals("org.h2.Driver", jdbcIcebergConfig.get(IcebergConfig.JDBC_DRIVER)); + Assertions.assertEquals(true, jdbcIcebergConfig.get(IcebergConfig.JDBC_INIT_TABLES)); + + Assertions.assertEquals( + defaultCatalogName, defaultIcebergConfig.get(IcebergConfig.CATALOG_BACKEND_NAME)); + Assertions.assertEquals("memory", defaultIcebergConfig.get(IcebergConfig.CATALOG_BACKEND)); + Assertions.assertEquals("/tmp/", defaultIcebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)); + + Assertions.assertEquals(hiveCatalogName, hiveOps.catalog.name()); + Assertions.assertEquals(jdbcCatalogName, jdbcOps.catalog.name()); + Assertions.assertEquals(defaultCatalogName, defaultOps.catalog.name()); + + Assertions.assertTrue(hiveOps.catalog instanceof HiveCatalog); + Assertions.assertTrue(jdbcOps.catalog instanceof JdbcCatalog); + Assertions.assertTrue(defaultOps.catalog instanceof InMemoryCatalog); + } + + @ParameterizedTest + @ValueSource(strings = {"", "not_match"}) + public void testInvalidIcebergTableOps(String catalogName) { + ConfigBasedIcebergTableOpsProvider provider = new ConfigBasedIcebergTableOpsProvider(); + provider.initialize(Maps.newHashMap()); + + Assertions.assertThrowsExactly( + RuntimeException.class, () -> provider.getIcebergTableOps(catalogName)); + } +} diff --git a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/ops/TestIcebergTableOpsManager.java b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/ops/TestIcebergTableOpsManager.java new file mode 100644 index 00000000000..04ded71dd7f --- /dev/null +++ b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/ops/TestIcebergTableOpsManager.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.common.ops; + +import com.google.common.collect.Maps; +import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class TestIcebergTableOpsManager { + + private static final String DEFAULT_CATALOG = "memory"; + + @ParameterizedTest + @ValueSource(strings = {"", "hello/", "\\\n\t\\\'/", "\u0024/", "\100/", "[_~/"}) + public void testValidGetOps(String rawPrefix) { + String prefix = rawPrefix; + if (!StringUtils.isBlank(rawPrefix)) { + prefix = rawPrefix.substring(0, rawPrefix.length() - 1); + } + Map config = Maps.newHashMap(); + config.put(String.format("catalog.%s.catalog-backend-name", prefix), prefix); + IcebergTableOpsManager manager = new IcebergTableOpsManager(config); + + IcebergTableOps ops = manager.getOps(rawPrefix); + + if (StringUtils.isBlank(prefix)) { + Assertions.assertEquals(ops.catalog.name(), DEFAULT_CATALOG); + } else { + Assertions.assertEquals(ops.catalog.name(), prefix); + } + } + + @ParameterizedTest + @ValueSource( + strings = {"hello", "\\\n\t\\\'", "\u0024", "\100", "[_~", "__gravitino_default_catalog/"}) + public void testInvalidGetOps(String rawPrefix) { + Map config = Maps.newHashMap(); + IcebergTableOpsManager manager = new IcebergTableOpsManager(config); + + Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> manager.getOps(rawPrefix)); + } +} diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index b289c72ed03..2c0fdb0091f 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -114,6 +114,8 @@ tasks { original } } + + fileMode = 0b111101101 } register("copyConfigsToStandalonePackage", Copy::class) { @@ -130,6 +132,8 @@ tasks { original } } + + fileMode = 0b111101101 } register("copyLibAndConfigs", Copy::class) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/RESTService.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/RESTService.java index 0a1bf752d9f..dd3b9d9abf8 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/RESTService.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/RESTService.java @@ -23,7 +23,7 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.auxiliary.GravitinoAuxiliaryService; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergTableOps; +import org.apache.gravitino.iceberg.common.ops.IcebergTableOpsManager; import org.apache.gravitino.iceberg.service.IcebergExceptionMapper; import org.apache.gravitino.iceberg.service.IcebergObjectMapperProvider; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; @@ -48,7 +48,7 @@ public class RESTService implements GravitinoAuxiliaryService { public static final String SERVICE_NAME = "iceberg-rest"; public static final String ICEBERG_SPEC = "/iceberg/*"; - private IcebergTableOps icebergTableOps; + private IcebergTableOpsManager icebergTableOpsManager; private IcebergMetricsManager icebergMetricsManager; private void initServer(IcebergConfig icebergConfig) { @@ -66,13 +66,13 @@ private void initServer(IcebergConfig icebergConfig) { new HttpServerMetricsSource(MetricsSource.ICEBERG_REST_SERVER_METRIC_NAME, config, server); metricsSystem.register(httpServerMetricsSource); - icebergTableOps = new IcebergTableOps(icebergConfig); + icebergTableOpsManager = new IcebergTableOpsManager(icebergConfig.getAllConfig()); icebergMetricsManager = new IcebergMetricsManager(icebergConfig); config.register( new AbstractBinder() { @Override protected void configure() { - bind(icebergTableOps).to(IcebergTableOps.class).ranked(1); + bind(icebergTableOpsManager).to(IcebergTableOpsManager.class).ranked(1); bind(icebergMetricsManager).to(IcebergMetricsManager.class).ranked(1); } }); @@ -114,8 +114,8 @@ public void serviceStop() throws Exception { server.stop(); LOG.info("Iceberg REST service stopped"); } - if (icebergTableOps != null) { - icebergTableOps.close(); + if (icebergTableOpsManager != null) { + icebergTableOpsManager.close(); } if (icebergMetricsManager != null) { icebergMetricsManager.close(); diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java index b37fab80b90..82b6250e61d 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergNamespaceOperations.java @@ -34,7 +34,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.gravitino.iceberg.common.ops.IcebergTableOps; +import org.apache.gravitino.iceberg.common.ops.IcebergTableOpsManager; import org.apache.gravitino.iceberg.service.IcebergRestUtils; import org.apache.gravitino.metrics.MetricNames; import org.apache.iceberg.catalog.Namespace; @@ -57,25 +57,27 @@ public class IcebergNamespaceOperations { private static final Logger LOG = LoggerFactory.getLogger(IcebergNamespaceOperations.class); - private IcebergTableOps icebergTableOps; + private IcebergTableOpsManager icebergTableOpsManager; @SuppressWarnings("UnusedVariable") @Context private HttpServletRequest httpRequest; @Inject - public IcebergNamespaceOperations(IcebergTableOps icebergTableOps) { - this.icebergTableOps = icebergTableOps; + public IcebergNamespaceOperations(IcebergTableOpsManager icebergTableOpsManager) { + this.icebergTableOpsManager = icebergTableOpsManager; } @GET @Produces(MediaType.APPLICATION_JSON) @Timed(name = "list-namespace." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "list-namespace", absolute = true) - public Response listNamespaces(@DefaultValue("") @QueryParam("parent") String parent) { + public Response listNamespaces( + @DefaultValue("") @QueryParam("parent") String parent, @PathParam("prefix") String prefix) { Namespace parentNamespace = parent.isEmpty() ? Namespace.empty() : RESTUtil.decodeNamespace(parent); - ListNamespacesResponse response = icebergTableOps.listNamespace(parentNamespace); + ListNamespacesResponse response = + icebergTableOpsManager.getOps(prefix).listNamespace(parentNamespace); return IcebergRestUtils.ok(response); } @@ -84,9 +86,10 @@ public Response listNamespaces(@DefaultValue("") @QueryParam("parent") String pa @Produces(MediaType.APPLICATION_JSON) @Timed(name = "load-namespace." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "load-namespace", absolute = true) - public Response loadNamespace(@PathParam("namespace") String namespace) { + public Response loadNamespace( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace) { GetNamespaceResponse getNamespaceResponse = - icebergTableOps.loadNamespace(RESTUtil.decodeNamespace(namespace)); + icebergTableOpsManager.getOps(prefix).loadNamespace(RESTUtil.decodeNamespace(namespace)); return IcebergRestUtils.ok(getNamespaceResponse); } @@ -95,10 +98,11 @@ public Response loadNamespace(@PathParam("namespace") String namespace) { @Produces(MediaType.APPLICATION_JSON) @Timed(name = "drop-namespace." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "drop-namespace", absolute = true) - public Response dropNamespace(@PathParam("namespace") String namespace) { + public Response dropNamespace( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace) { // todo check if table exists in namespace after table ops is added - LOG.info("Drop Iceberg namespace: {}", namespace); - icebergTableOps.dropNamespace(RESTUtil.decodeNamespace(namespace)); + LOG.info("Drop Iceberg namespace: {}, prefix: {}", namespace, prefix); + icebergTableOpsManager.getOps(prefix).dropNamespace(RESTUtil.decodeNamespace(namespace)); return IcebergRestUtils.noContent(); } @@ -106,9 +110,11 @@ public Response dropNamespace(@PathParam("namespace") String namespace) { @Produces(MediaType.APPLICATION_JSON) @Timed(name = "create-namespace." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "create-namespace", absolute = true) - public Response createNamespace(CreateNamespaceRequest namespaceRequest) { - LOG.info("Create Iceberg namespace: {}", namespaceRequest); - CreateNamespaceResponse response = icebergTableOps.createNamespace(namespaceRequest); + public Response createNamespace( + @PathParam("prefix") String prefix, CreateNamespaceRequest namespaceRequest) { + LOG.info("Create Iceberg namespace: {}, prefix: {}", namespaceRequest, prefix); + CreateNamespaceResponse response = + icebergTableOpsManager.getOps(prefix).createNamespace(namespaceRequest); return IcebergRestUtils.ok(response); } @@ -118,10 +124,14 @@ public Response createNamespace(CreateNamespaceRequest namespaceRequest) { @Timed(name = "update-namespace." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "update-namespace", absolute = true) public Response updateNamespace( - @PathParam("namespace") String namespace, UpdateNamespacePropertiesRequest request) { - LOG.info("Update Iceberg namespace: {}, request: {}", namespace, request); + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + UpdateNamespacePropertiesRequest request) { + LOG.info("Update Iceberg namespace: {}, request: {}, prefix: {}", namespace, request, prefix); UpdateNamespacePropertiesResponse response = - icebergTableOps.updateNamespaceProperties(RESTUtil.decodeNamespace(namespace), request); + icebergTableOpsManager + .getOps(prefix) + .updateNamespaceProperties(RESTUtil.decodeNamespace(namespace), request); return IcebergRestUtils.ok(response); } @@ -131,10 +141,14 @@ public Response updateNamespace( @Timed(name = "register-table." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "register-table", absolute = true) public Response registerTable( - @PathParam("namespace") String namespace, RegisterTableRequest request) { + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + RegisterTableRequest request) { LOG.info("Register table, namespace: {}, request: {}", namespace, request); LoadTableResponse response = - icebergTableOps.registerTable(RESTUtil.decodeNamespace(namespace), request); + icebergTableOpsManager + .getOps(prefix) + .registerTable(RESTUtil.decodeNamespace(namespace), request); return IcebergRestUtils.ok(response); } } diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java index 0b35f45c7d2..f6f6042a5b2 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java @@ -37,7 +37,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.gravitino.iceberg.common.ops.IcebergTableOps; +import org.apache.gravitino.iceberg.common.ops.IcebergTableOpsManager; import org.apache.gravitino.iceberg.service.IcebergObjectMapper; import org.apache.gravitino.iceberg.service.IcebergRestUtils; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; @@ -57,7 +57,7 @@ public class IcebergTableOperations { private static final Logger LOG = LoggerFactory.getLogger(IcebergTableOperations.class); - private IcebergTableOps icebergTableOps; + private IcebergTableOpsManager icebergTableOpsManager; private IcebergMetricsManager icebergMetricsManager; private ObjectMapper icebergObjectMapper; @@ -68,8 +68,8 @@ public class IcebergTableOperations { @Inject public IcebergTableOperations( - IcebergTableOps icebergTableOps, IcebergMetricsManager icebergMetricsManager) { - this.icebergTableOps = icebergTableOps; + IcebergTableOpsManager icebergTableOpsManager, IcebergMetricsManager icebergMetricsManager) { + this.icebergTableOpsManager = icebergTableOpsManager; this.icebergObjectMapper = IcebergObjectMapper.getInstance(); this.icebergMetricsManager = icebergMetricsManager; } @@ -78,8 +78,10 @@ public IcebergTableOperations( @Produces(MediaType.APPLICATION_JSON) @Timed(name = "list-table." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "list-table", absolute = true) - public Response listTable(@PathParam("namespace") String namespace) { - return IcebergRestUtils.ok(icebergTableOps.listTable(RESTUtil.decodeNamespace(namespace))); + public Response listTable( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace) { + return IcebergRestUtils.ok( + icebergTableOpsManager.getOps(prefix).listTable(RESTUtil.decodeNamespace(namespace))); } @POST @@ -87,13 +89,17 @@ public Response listTable(@PathParam("namespace") String namespace) { @Timed(name = "create-table." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "create-table", absolute = true) public Response createTable( - @PathParam("namespace") String namespace, CreateTableRequest createTableRequest) { + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + CreateTableRequest createTableRequest) { LOG.info( "Create Iceberg table, namespace: {}, create table request: {}", namespace, createTableRequest); return IcebergRestUtils.ok( - icebergTableOps.createTable(RESTUtil.decodeNamespace(namespace), createTableRequest)); + icebergTableOpsManager + .getOps(prefix) + .createTable(RESTUtil.decodeNamespace(namespace), createTableRequest)); } @POST @@ -102,6 +108,7 @@ public Response createTable( @Timed(name = "update-table." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "update-table", absolute = true) public Response updateTable( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, @PathParam("table") String table, UpdateTableRequest updateTableRequest) { @@ -114,7 +121,8 @@ public Response updateTable( } TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), table); - return IcebergRestUtils.ok(icebergTableOps.updateTable(tableIdentifier, updateTableRequest)); + return IcebergRestUtils.ok( + icebergTableOpsManager.getOps(prefix).updateTable(tableIdentifier, updateTableRequest)); } @DELETE @@ -123,6 +131,7 @@ public Response updateTable( @Timed(name = "drop-table." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "drop-table", absolute = true) public Response dropTable( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, @PathParam("table") String table, @DefaultValue("false") @QueryParam("purgeRequested") boolean purgeRequested) { @@ -134,9 +143,9 @@ public Response dropTable( TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), table); if (purgeRequested) { - icebergTableOps.purgeTable(tableIdentifier); + icebergTableOpsManager.getOps(prefix).purgeTable(tableIdentifier); } else { - icebergTableOps.dropTable(tableIdentifier); + icebergTableOpsManager.getOps(prefix).dropTable(tableIdentifier); } return IcebergRestUtils.noContent(); } @@ -147,13 +156,14 @@ public Response dropTable( @Timed(name = "load-table." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "load-table", absolute = true) public Response loadTable( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, @PathParam("table") String table, @DefaultValue("all") @QueryParam("snapshots") String snapshots) { // todo support snapshots TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), table); - return IcebergRestUtils.ok(icebergTableOps.loadTable(tableIdentifier)); + return IcebergRestUtils.ok(icebergTableOpsManager.getOps(prefix).loadTable(tableIdentifier)); } @HEAD @@ -162,10 +172,12 @@ public Response loadTable( @Timed(name = "table-exists." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "table-exits", absolute = true) public Response tableExists( - @PathParam("namespace") String namespace, @PathParam("table") String table) { + @PathParam("prefix") String prefix, + @PathParam("namespace") String namespace, + @PathParam("table") String table) { TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), table); - if (icebergTableOps.tableExists(tableIdentifier)) { + if (icebergTableOpsManager.getOps(prefix).tableExists(tableIdentifier)) { return IcebergRestUtils.okWithoutContent(); } else { return IcebergRestUtils.notExists(); @@ -178,6 +190,7 @@ public Response tableExists( @Timed(name = "report-table-metrics." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "report-table-metrics", absolute = true) public Response reportTableMetrics( + @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, @PathParam("table") String table, ReportMetricsRequest request) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableRenameOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableRenameOperations.java index 2b8585ba64a..441ce0551ba 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableRenameOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableRenameOperations.java @@ -25,11 +25,12 @@ import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.gravitino.iceberg.common.ops.IcebergTableOps; +import org.apache.gravitino.iceberg.common.ops.IcebergTableOpsManager; import org.apache.gravitino.iceberg.service.IcebergRestUtils; import org.apache.gravitino.metrics.MetricNames; import org.apache.iceberg.rest.requests.RenameTableRequest; @@ -43,19 +44,20 @@ public class IcebergTableRenameOperations { @Context private HttpServletRequest httpRequest; - private IcebergTableOps icebergTableOps; + private IcebergTableOpsManager icebergTableOpsManager; @Inject - public IcebergTableRenameOperations(IcebergTableOps icebergTableOps) { - this.icebergTableOps = icebergTableOps; + public IcebergTableRenameOperations(IcebergTableOpsManager icebergTableOpsManager) { + this.icebergTableOpsManager = icebergTableOpsManager; } @POST @Produces(MediaType.APPLICATION_JSON) @Timed(name = "rename-table." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "rename-table", absolute = true) - public Response renameTable(RenameTableRequest renameTableRequest) { - icebergTableOps.renameTable(renameTableRequest); + public Response renameTable( + @PathParam("prefix") String prefix, RenameTableRequest renameTableRequest) { + icebergTableOpsManager.getOps(prefix).renameTable(renameTableRequest); return IcebergRestUtils.okWithoutContent(); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java index 837b4096185..2284a9a8257 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java @@ -32,7 +32,7 @@ @Tag("gravitino-docker-test") @TestInstance(Lifecycle.PER_CLASS) public class IcebergRESTHiveCatalogIT extends IcebergRESTServiceIT { - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); public IcebergRESTHiveCatalogIT() { catalogType = IcebergCatalogBackend.HIVE; diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveCatalogIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveCatalogIT.java new file mode 100644 index 00000000000..e647c59597b --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRestKerberosHiveCatalogIT.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.integration.test; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.commons.io.FileUtils; +import org.apache.gravitino.iceberg.common.IcebergCatalogBackend; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.gravitino.integration.test.util.ITUtils; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.api.condition.EnabledIf; + +@Tag("gravitino-docker-test") +@TestInstance(Lifecycle.PER_CLASS) +@EnabledIf("isEmbedded") +public class IcebergRestKerberosHiveCatalogIT extends IcebergRESTHiveCatalogIT { + + private static final String HIVE_METASTORE_CLIENT_PRINCIPAL = "cli@HADOOPKRB"; + private static final String HIVE_METASTORE_CLIENT_KEYTAB = "/client.keytab"; + + private static String tempDir; + + public IcebergRestKerberosHiveCatalogIT() { + super(); + } + + void initEnv() { + containerSuite.startKerberosHiveContainer(); + try { + + // Init kerberos configurations; + File baseDir = new File(System.getProperty("java.io.tmpdir")); + File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); + file.deleteOnExit(); + tempDir = file.getAbsolutePath(); + + HiveContainer kerberosHiveContainer = containerSuite.getKerberosHiveContainer(); + kerberosHiveContainer + .getContainer() + .copyFileFromContainer("/etc/admin.keytab", tempDir + HIVE_METASTORE_CLIENT_KEYTAB); + + String tmpKrb5Path = tempDir + "/krb5.conf_tmp"; + String krb5Path = tempDir + "/krb5.conf"; + kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path); + + // Modify the krb5.conf and change the kdc and admin_server to the container IP + String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8); + content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); + content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); + FileUtils.write(new File(krb5Path), content, StandardCharsets.UTF_8); + + LOG.info("Kerberos kdc config:\n{}, path: {}", content, krb5Path); + System.setProperty("java.security.krb5.conf", krb5Path); + System.setProperty("sun.security.krb5.debug", "true"); + System.setProperty("java.security.krb5.realm", "HADOOPKRB"); + System.setProperty("java.security.krb5.kdc", ip); + + sun.security.krb5.Config.refresh(); + resetDefaultRealm(); + + // Give cli@HADOOPKRB permission to access the hdfs + containerSuite + .getKerberosHiveContainer() + .executeInContainer("hadoop", "fs", "-chown", "-R", "cli", "/user/hive/"); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + void resetDefaultRealm() { + try { + String kerberosNameClass = "org.apache.hadoop.security.authentication.util.KerberosName"; + Class cl = Class.forName(kerberosNameClass); + cl.getMethod("resetDefaultRealm").invoke(null); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + Map getCatalogConfig() { + Map configMap = new HashMap<>(); + + configMap.put("gravitino.iceberg-rest.authentication.type", "kerberos"); + configMap.put( + "gravitino.iceberg-rest.authentication.kerberos.principal", + HIVE_METASTORE_CLIENT_PRINCIPAL); + configMap.put( + "gravitino.iceberg-rest.authentication.kerberos.keytab-uri", + tempDir + HIVE_METASTORE_CLIENT_KEYTAB); + configMap.put("gravitino.iceberg-rest.hive.metastore.sasl.enabled", "true"); + configMap.put( + "gravitino.iceberg-rest.hive.metastore.kerberos.principal", + "hive/_HOST@HADOOPKRB" + .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + + configMap.put("gravitino.iceberg-rest.hadoop.security.authentication", "kerberos"); + + configMap.put( + "gravitino.iceberg-rest.dfs.namenode.kerberos.principal", + "hdfs/_HOST@HADOOPKRB" + .replace("_HOST", containerSuite.getKerberosHiveContainer().getHostName())); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_BACKEND.getKey(), + IcebergCatalogBackend.HIVE.toString().toLowerCase()); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_URI.getKey(), + String.format( + "thrift://%s:%d", + containerSuite.getKerberosHiveContainer().getContainerIpAddress(), + HiveContainer.HIVE_METASTORE_PORT)); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(), + GravitinoITUtils.genRandomName( + String.format( + "hdfs://%s:%d/user/hive/warehouse-hive", + containerSuite.getKerberosHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT))); + return configMap; + } + + private static boolean isEmbedded() { + String mode = + System.getProperty(ITUtils.TEST_MODE) == null + ? ITUtils.EMBEDDED_TEST_MODE + : System.getProperty(ITUtils.TEST_MODE); + + return Objects.equals(mode, ITUtils.EMBEDDED_TEST_MODE); + } +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManager.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManager.java index 6e2aa5c2852..c70888d7bbb 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManager.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManager.java @@ -123,7 +123,6 @@ private void customizeConfigFile(String configTempFileName, String configFileNam String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); configMap.putAll(customConfigs); - ITUtils.rewriteConfigFile(configTempFileName, configFileName, configMap); } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManagerForDeploy.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManagerForDeploy.java index dda2c593ef5..135713223ab 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManagerForDeploy.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/util/IcebergRESTServerManagerForDeploy.java @@ -18,10 +18,14 @@ */ package org.apache.gravitino.iceberg.integration.test.util; +import java.io.File; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.Future; +import org.apache.commons.io.FileUtils; import org.apache.gravitino.integration.test.util.CommandExecutor; import org.apache.gravitino.integration.test.util.JdbcDriverDownloader; import org.apache.gravitino.integration.test.util.ProcessData; @@ -51,12 +55,39 @@ public Optional> doStartIcebergRESTServer() throws Exception { SQLITE_DRIVER_DOWNLOAD_URL, Paths.get(icebergRESTServerHome.toString(), "iceberg-rest-server", "libs").toString()); - String cmd = String.format("%s/bin/%s start", icebergRESTServerHome.toString(), SCRIPT_NAME); - CommandExecutor.executeCommandLocalHost( - cmd, - false, - ProcessData.TypesOfData.OUTPUT, - ImmutableMap.of("GRAVITINO_HOME", icebergRESTServerHome.toString())); + String gravitinoRestStartShell = icebergRESTServerHome.toString() + "/bin/" + SCRIPT_NAME; + String krb5Path = System.getProperty("java.security.krb5.conf"); + if (krb5Path != null) { + LOG.info("java.security.krb5.conf: {}", krb5Path); + String modifiedGravitinoStartShell = + String.format( + "%s/bin/gravitino-iceberg-rest-server_%s.sh", + icebergRESTServerHome.toString(), UUID.randomUUID()); + // Replace '/etc/krb5.conf' with the one in the test + try { + String content = + FileUtils.readFileToString(new File(gravitinoRestStartShell), StandardCharsets.UTF_8); + content = + content.replace( + "#JAVA_OPTS+=\" -Djava.security.krb5.conf=/etc/krb5.conf\"", + String.format("JAVA_OPTS+=\" -Djava.security.krb5.conf=%s\"", krb5Path)); + File tmp = new File(modifiedGravitinoStartShell); + FileUtils.write(tmp, content, StandardCharsets.UTF_8); + tmp.setExecutable(true); + LOG.info("modifiedGravitinoStartShell content: \n{}", content); + CommandExecutor.executeCommandLocalHost( + modifiedGravitinoStartShell + " start", false, ProcessData.TypesOfData.OUTPUT); + } catch (Exception e) { + LOG.error("Can replace /etc/krb5.conf with real kerberos configuration", e); + } + } else { + String cmd = String.format("%s/bin/%s start", icebergRESTServerHome.toString(), SCRIPT_NAME); + CommandExecutor.executeCommandLocalHost( + cmd, + false, + ProcessData.TypesOfData.OUTPUT, + ImmutableMap.of("GRAVITINO_HOME", icebergRESTServerHome.toString())); + } return Optional.empty(); } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergTableOpsProviderForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergTableOpsProviderForTest.java new file mode 100644 index 00000000000..6a937561551 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergTableOpsProviderForTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.iceberg.service.rest; + +import org.apache.gravitino.iceberg.common.ops.ConfigBasedIcebergTableOpsProvider; +import org.apache.gravitino.iceberg.common.ops.IcebergTableOps; + +public class ConfigBasedIcebergTableOpsProviderForTest extends ConfigBasedIcebergTableOpsProvider { + @Override + public IcebergTableOps getIcebergTableOps(String prefix) { + return new IcebergTableOpsForTest(); + } +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java index 0e3d29e4975..e7b19e0aff5 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java @@ -19,10 +19,13 @@ package org.apache.gravitino.iceberg.service.rest; +import com.google.common.collect.Maps; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergTableOps; +import org.apache.gravitino.iceberg.common.ops.IcebergTableOpsManager; import org.apache.gravitino.iceberg.service.IcebergExceptionMapper; import org.apache.gravitino.iceberg.service.IcebergObjectMapperProvider; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; @@ -66,13 +69,19 @@ public static ResourceConfig getIcebergResourceConfig(Class c, boolean bindIcebe } if (bindIcebergTableOps) { - IcebergTableOps icebergTableOps = new IcebergTableOpsForTest(); + Map catalogConf = Maps.newHashMap(); + catalogConf.put(String.format("catalog.%s.catalog-backend-name", PREFIX), PREFIX); + catalogConf.put( + IcebergConstants.ICEBERG_REST_CATALOG_PROVIDER, + ConfigBasedIcebergTableOpsProviderForTest.class.getName()); + IcebergTableOpsManager icebergTableOpsManager = new IcebergTableOpsManager(catalogConf); + IcebergMetricsManager icebergMetricsManager = new IcebergMetricsManager(new IcebergConfig()); resourceConfig.register( new AbstractBinder() { @Override protected void configure() { - bind(icebergTableOps).to(IcebergTableOps.class).ranked(2); + bind(icebergTableOpsManager).to(IcebergTableOpsManager.class).ranked(2); bind(icebergMetricsManager).to(IcebergMetricsManager.class).ranked(2); } }); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java index ee672bf093f..7d1d80b54e3 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergTestBase.java @@ -27,6 +27,7 @@ import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.iceberg.service.IcebergObjectMapperProvider; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.test.JerseyTest; @@ -38,7 +39,7 @@ protected Application configure() { return new ResourceConfig(); } - private boolean urlPathWithPrefix = false; + private String urlPathPrefix = ""; public Invocation.Builder getRenameTableClientBuilder() { return getIcebergClientBuilder(IcebergRestTestUtil.RENAME_TABLE_PATH, Optional.empty()); @@ -109,8 +110,8 @@ public String injectPrefixToPath(String path, String prefix) { public Invocation.Builder getIcebergClientBuilder( String path, Optional> queryParam) { - if (urlPathWithPrefix) { - path = injectPrefixToPath(path, IcebergRestTestUtil.PREFIX); + if (!StringUtils.isBlank(urlPathPrefix)) { + path = injectPrefixToPath(path, urlPathPrefix); } WebTarget target = target(path); if (queryParam.isPresent()) { @@ -126,7 +127,7 @@ public Invocation.Builder getIcebergClientBuilder( .accept(MediaType.APPLICATION_JSON_TYPE); } - public void setUrlPathWithPrefix(boolean urlPathWithPrefix) { - this.urlPathWithPrefix = urlPathWithPrefix; + public void setUrlPathWithPrefix(String urlPathPrefix) { + this.urlPathPrefix = urlPathPrefix; } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergConfig.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergConfig.java index b2f9e09efb4..1116fc0bb0d 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergConfig.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergConfig.java @@ -32,13 +32,13 @@ public class TestIcebergConfig extends IcebergTestBase { @Override protected Application configure() { - return IcebergRestTestUtil.getIcebergResourceConfig(IcebergConfigOperations.class, false); + return IcebergRestTestUtil.getIcebergResourceConfig(IcebergConfigOperations.class); } @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testConfig(boolean withPrefix) { - setUrlPathWithPrefix(withPrefix); + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + public void testConfig(String prefix) { + setUrlPathWithPrefix(prefix); Response resp = getConfigClientBuilder().get(); Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java index 251f9e90709..6049a153e1c 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergNamespaceOperations.java @@ -209,9 +209,9 @@ private void verifyListNamespaceSucc(Optional parent, List schem } @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testListNamespace(boolean withPrefix) { - setUrlPathWithPrefix(withPrefix); + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListNamespace(String prefix) { + setUrlPathWithPrefix(prefix); dropAllExistingNamespace(); verifyListNamespaceSucc(Optional.empty(), Arrays.asList()); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java index e2d49d98528..6037302b8b2 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java @@ -260,9 +260,9 @@ void testUpdateTable() { } @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testListTables(boolean withPrefix) { - setUrlPathWithPrefix(withPrefix); + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListTables(String prefix) { + setUrlPathWithPrefix(prefix); verifyListTableFail(404); verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); @@ -283,9 +283,9 @@ void testTableExits() { } @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testRenameTable(boolean withPrefix) { - setUrlPathWithPrefix(withPrefix); + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testRenameTable(String prefix) { + setUrlPathWithPrefix(prefix); // namespace not exits verifyRenameTableFail("rename_foo1", "rename_foo3", 404); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java index 00286d04505..4f477e13649 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java @@ -45,6 +45,7 @@ public class ContainerSuite implements Closeable { public static final Logger LOG = LoggerFactory.getLogger(ContainerSuite.class); private static volatile ContainerSuite instance = null; + private static volatile boolean initialized = false; // The subnet must match the configuration in // `dev/docker/tools/mac-docker-connector.conf` @@ -72,7 +73,11 @@ public class ContainerSuite implements Closeable { protected static final CloseableGroup closer = CloseableGroup.create(); - private static void init() { + private static void initIfNecessary() { + if (initialized) { + return; + } + try { // Check if docker is available and you should never close the global DockerClient! DockerClient dockerClient = DockerClientFactory.instance().client(); @@ -82,16 +87,20 @@ private static void init() { if ("true".equalsIgnoreCase(System.getenv("NEED_CREATE_DOCKER_NETWORK"))) { network = createDockerNetwork(); } + initialized = true; } catch (Exception e) { throw new RuntimeException("Failed to initialize ContainerSuite", e); } } + public static boolean initialized() { + return initialized; + } + public static ContainerSuite getInstance() { if (instance == null) { synchronized (ContainerSuite.class) { if (instance == null) { - init(); instance = new ContainerSuite(); } } @@ -107,6 +116,7 @@ public void startHiveContainer() { if (hiveContainer == null) { synchronized (ContainerSuite.class) { if (hiveContainer == null) { + initIfNecessary(); // Start Hive container HiveContainer.Builder hiveBuilder = HiveContainer.builder() @@ -167,6 +177,7 @@ public void startKerberosHiveContainer() { if (kerberosHiveContainer == null) { synchronized (ContainerSuite.class) { if (kerberosHiveContainer == null) { + initIfNecessary(); // Start Hive container HiveContainer.Builder hiveBuilder = HiveContainer.builder() @@ -189,6 +200,7 @@ public void startTrinoContainer( if (trinoContainer == null) { synchronized (ContainerSuite.class) { if (trinoContainer == null) { + initIfNecessary(); // Start Trino container String hiveContainerIp = hiveContainer.getContainerIpAddress(); TrinoContainer.Builder trinoBuilder = @@ -229,6 +241,7 @@ public void startDorisContainer() { if (dorisContainer == null) { synchronized (ContainerSuite.class) { if (dorisContainer == null) { + initIfNecessary(); // Start Doris container DorisContainer.Builder dorisBuilder = DorisContainer.builder().withHostName("gravitino-ci-doris").withNetwork(network); @@ -244,6 +257,7 @@ public void startMySQLContainer(TestDatabaseName testDatabaseName) { if (mySQLContainer == null) { synchronized (ContainerSuite.class) { if (mySQLContainer == null) { + initIfNecessary(); // Start MySQL container MySQLContainer.Builder mysqlBuilder = MySQLContainer.builder() @@ -270,6 +284,7 @@ public void startMySQLVersion5Container(TestDatabaseName testDatabaseName) { if (mySQLVersion5Container == null) { synchronized (ContainerSuite.class) { if (mySQLVersion5Container == null) { + initIfNecessary(); // Start MySQL container MySQLContainer.Builder mysqlBuilder = MySQLContainer.builder() @@ -297,6 +312,7 @@ public void startPostgreSQLContainer(TestDatabaseName testDatabaseName, PGImageN if (!pgContainerMap.containsKey(pgImageName)) { synchronized (ContainerSuite.class) { if (!pgContainerMap.containsKey(pgImageName)) { + initIfNecessary(); // Start PostgreSQL container PostgreSQLContainer.Builder pgBuilder = PostgreSQLContainer.builder() @@ -330,6 +346,7 @@ public void startKafkaContainer() { if (kafkaContainer == null) { synchronized (ContainerSuite.class) { if (kafkaContainer == null) { + initIfNecessary(); KafkaContainer.Builder builder = KafkaContainer.builder().withNetwork(network); KafkaContainer container = closer.register(builder.build()); try { @@ -372,6 +389,7 @@ public void startRangerContainer() { if (rangerContainer == null) { synchronized (ContainerSuite.class) { if (rangerContainer == null) { + initIfNecessary(); // Start Ranger container RangerContainer.Builder rangerBuilder = RangerContainer.builder().withNetwork(network); RangerContainer container = closer.register(rangerBuilder.build()); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java index f3b3a5b5a10..82d823f0e25 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseContainerExtension.java @@ -33,12 +33,13 @@ public class CloseContainerExtension implements BeforeAllCallback { @Override public void beforeAll(ExtensionContext extensionContext) { // Ensure that the container suite is initialized before closing it - ContainerSuite.getInstance(); - synchronized (CloseContainerExtension.class) { - extensionContext - .getRoot() - .getStore(ExtensionContext.Namespace.GLOBAL) - .getOrComputeIfAbsent(CloseableContainer.class); + if (ContainerSuite.initialized()) { + synchronized (CloseContainerExtension.class) { + extensionContext + .getRoot() + .getStore(ExtensionContext.Namespace.GLOBAL) + .getOrComputeIfAbsent(CloseableContainer.class); + } } } diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/TagIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/TagIT.java index bb650b3e713..e0cb7326982 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/TagIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/TagIT.java @@ -182,6 +182,16 @@ public void testCreateGetAndListTag() { Set tags = Sets.newHashSet(metalake.listTagsInfo()); Set expectedTags = Sets.newHashSet(tag, tag1); Assertions.assertEquals(expectedTags, tags); + + // Test null comment and properties + String tagName2 = GravitinoITUtils.genRandomName("tag_it_tag2"); + Tag tag2 = metalake.createTag(tagName2, null, null); + Assertions.assertEquals(tagName2, tag2.name()); + Assertions.assertNull(tag2.comment()); + Assertions.assertEquals(Collections.emptyMap(), tag2.properties()); + + Tag tag3 = metalake.getTag(tagName2); + Assertions.assertEquals(tag2, tag3); } @Test @@ -228,6 +238,26 @@ public void testCreateAndAlterTag() { // Test throw NoSuchTagException Assertions.assertThrows( NoSuchTagException.class, () -> metalake.alterTag("non-existed-tag", rename)); + + // Test alter tag on no comment and properties + String tagName1 = GravitinoITUtils.genRandomName("tag_it_tag1"); + metalake.createTag(tagName1, null, null); + + // Test rename and update comment + String newTagName1 = GravitinoITUtils.genRandomName("tag_it_tag_new1"); + TagChange rename1 = TagChange.rename(newTagName1); + TagChange updateComment1 = TagChange.updateComment("new comment1"); + TagChange setProperty3 = TagChange.setProperty("k4", "v4"); + TagChange setProperty4 = TagChange.setProperty("k5", "v5"); + TagChange removeProperty3 = TagChange.removeProperty("k4"); + + Tag alteredTag5 = + metalake.alterTag( + tagName1, rename1, updateComment1, setProperty3, setProperty4, removeProperty3); + Assertions.assertEquals(newTagName1, alteredTag5.name()); + Assertions.assertEquals("new comment1", alteredTag5.comment()); + Assertions.assertEquals(ImmutableMap.of("k5", "v5"), alteredTag5.properties()); + Assertions.assertFalse(alteredTag5.inherited().isPresent()); } @Test diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/AccessControlIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/AccessControlIT.java new file mode 100644 index 00000000000..62366d07504 --- /dev/null +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/AccessControlIT.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.integration.test.authorization; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import org.apache.gravitino.Configs; +import org.apache.gravitino.auth.AuthConstants; +import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.User; +import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchGroupException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.NoSuchRoleException; +import org.apache.gravitino.exceptions.NoSuchUserException; +import org.apache.gravitino.exceptions.UserAlreadyExistsException; +import org.apache.gravitino.integration.test.util.AbstractIT; +import org.apache.gravitino.utils.RandomNameUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class AccessControlIT extends AbstractIT { + + private static String metalakeName = RandomNameUtils.genRandomName("metalake"); + private static GravitinoMetalake metalake; + + @BeforeAll + public static void startIntegrationTest() throws Exception { + Map configs = Maps.newHashMap(); + configs.put(Configs.ENABLE_AUTHORIZATION.getKey(), String.valueOf(true)); + configs.put(Configs.SERVICE_ADMINS.getKey(), AuthConstants.ANONYMOUS_USER); + registerCustomConfigs(configs); + AbstractIT.startIntegrationTest(); + metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); + } + + @Test + void testManageUsers() { + String username = "user1#123"; + User user = metalake.addUser(username); + Assertions.assertEquals(username, user.name()); + Assertions.assertTrue(user.roles().isEmpty()); + + // Add an existed user + Assertions.assertThrows(UserAlreadyExistsException.class, () -> metalake.addUser(username)); + + user = metalake.getUser(username); + Assertions.assertEquals(username, user.name()); + Assertions.assertTrue(user.roles().isEmpty()); + + // Get a not-existed user + Assertions.assertThrows(NoSuchUserException.class, () -> metalake.getUser("not-existed")); + + Assertions.assertTrue(metalake.removeUser(username)); + Assertions.assertFalse(metalake.removeUser(username)); + } + + @Test + void testManageGroups() { + String groupName = "group1#123"; + Group group = metalake.addGroup(groupName); + Assertions.assertEquals(group.name(), groupName); + Assertions.assertTrue(group.roles().isEmpty()); + + // Added an existed group + Assertions.assertThrows(GroupAlreadyExistsException.class, () -> metalake.addGroup(groupName)); + + group = metalake.getGroup(groupName); + Assertions.assertEquals(group.name(), groupName); + Assertions.assertTrue(group.roles().isEmpty()); + + // Get a not-existed group + Assertions.assertThrows(NoSuchGroupException.class, () -> metalake.getGroup("not-existed")); + + Assertions.assertTrue(metalake.removeGroup(groupName)); + Assertions.assertFalse(metalake.removeGroup(groupName)); + } + + @Test + void testManageRoles() { + String roleName = "role#123"; + Map properties = Maps.newHashMap(); + properties.put("k1", "v1"); + SecurableObject metalakeObject = + SecurableObjects.ofMetalake( + metalakeName, Lists.newArrayList(Privileges.CreateCatalog.allow())); + Role role = metalake.createRole(roleName, properties, Lists.newArrayList(metalakeObject)); + + Assertions.assertEquals(roleName, role.name()); + Assertions.assertEquals(properties, role.properties()); + + // Verify the object + Assertions.assertEquals(1, role.securableObjects().size()); + SecurableObject createdObject = role.securableObjects().get(0); + Assertions.assertEquals(metalakeObject.name(), createdObject.name()); + Assertions.assertEquals(metalakeObject.type(), createdObject.type()); + + // Verify the privilege + Assertions.assertEquals(1, createdObject.privileges().size()); + Privilege createdPrivilege = createdObject.privileges().get(0); + Assertions.assertEquals(createdPrivilege.name(), Privilege.Name.CREATE_CATALOG); + Assertions.assertEquals(createdPrivilege.condition(), Privilege.Condition.ALLOW); + + // Test a not-existed metadata object + SecurableObject catalogObject = + SecurableObjects.ofCatalog( + "not-existed", Lists.newArrayList(Privileges.UseCatalog.allow())); + + Assertions.assertThrows( + NoSuchMetadataObjectException.class, + () -> metalake.createRole("not-existed", properties, Lists.newArrayList(catalogObject))); + + // Get a role + role = metalake.getRole(roleName); + + Assertions.assertEquals(roleName, role.name()); + Assertions.assertEquals(properties, role.properties()); + + // Verify the object + Assertions.assertEquals(1, role.securableObjects().size()); + createdObject = role.securableObjects().get(0); + Assertions.assertEquals(metalakeObject.name(), createdObject.name()); + Assertions.assertEquals(metalakeObject.type(), createdObject.type()); + + // Verify the privilege + Assertions.assertEquals(1, createdObject.privileges().size()); + createdPrivilege = createdObject.privileges().get(0); + Assertions.assertEquals(createdPrivilege.name(), Privilege.Name.CREATE_CATALOG); + Assertions.assertEquals(createdPrivilege.condition(), Privilege.Condition.ALLOW); + + // Delete a role + Assertions.assertTrue(metalake.deleteRole(roleName)); + Assertions.assertFalse(metalake.deleteRole(roleName)); + } + + @Test + void testManageUserPermissions() { + String username = "user1#123"; + User user = metalake.addUser(username); + Assertions.assertTrue(user.roles().isEmpty()); + + String roleName = "role#123"; + Map properties = Maps.newHashMap(); + properties.put("k1", "v1"); + SecurableObject metalakeObject = + SecurableObjects.ofMetalake( + metalakeName, Lists.newArrayList(Privileges.CreateCatalog.allow())); + metalake.createRole(roleName, properties, Lists.newArrayList(metalakeObject)); + + user = metalake.grantRolesToUser(Lists.newArrayList(roleName), username); + Assertions.assertEquals(Lists.newArrayList(roleName), user.roles()); + + user = metalake.revokeRolesFromUser(Lists.newArrayList(roleName), username); + Assertions.assertTrue(user.roles().isEmpty()); + + // Grant a not-existed role + Assertions.assertThrows( + NoSuchRoleException.class, + () -> metalake.grantRolesToUser(Lists.newArrayList("not-existed"), username)); + + // Revoke a not-existed role + Assertions.assertThrows( + NoSuchRoleException.class, + () -> metalake.revokeRolesFromUser(Lists.newArrayList("not-existed"), username)); + + // Grant to a not-existed user + Assertions.assertThrows( + NoSuchUserException.class, + () -> metalake.grantRolesToUser(Lists.newArrayList(roleName), "not-existed")); + + // Revoke from a not-existed user + Assertions.assertThrows( + NoSuchUserException.class, + () -> metalake.grantRolesToUser(Lists.newArrayList(roleName), "not-existed")); + + // Grant a granted role + metalake.grantRolesToUser(Lists.newArrayList(roleName), username); + Assertions.assertDoesNotThrow( + () -> metalake.grantRolesToUser(Lists.newArrayList(roleName), username)); + + // Revoke a revoked role + metalake.revokeRolesFromUser(Lists.newArrayList(roleName), username); + Assertions.assertDoesNotThrow( + () -> metalake.revokeRolesFromUser(Lists.newArrayList(roleName), username)); + + // Clean up + metalake.removeUser(username); + metalake.deleteRole(roleName); + } + + @Test + void testManageGroupPermissions() { + String groupName = "group1#123"; + Group group = metalake.addGroup(groupName); + Assertions.assertTrue(group.roles().isEmpty()); + + String roleName = "role#123"; + Map properties = Maps.newHashMap(); + properties.put("k1", "v1"); + SecurableObject metalakeObject = + SecurableObjects.ofMetalake( + metalakeName, Lists.newArrayList(Privileges.CreateCatalog.allow())); + metalake.createRole(roleName, properties, Lists.newArrayList(metalakeObject)); + + group = metalake.grantRolesToGroup(Lists.newArrayList(roleName), groupName); + Assertions.assertEquals(Lists.newArrayList(roleName), group.roles()); + + group = metalake.revokeRolesFromGroup(Lists.newArrayList(roleName), groupName); + Assertions.assertTrue(group.roles().isEmpty()); + + // Grant a not-existed role + Assertions.assertThrows( + NoSuchRoleException.class, + () -> metalake.grantRolesToGroup(Lists.newArrayList("not-existed"), groupName)); + + // Revoke a not-existed role + Assertions.assertThrows( + NoSuchRoleException.class, + () -> metalake.revokeRolesFromGroup(Lists.newArrayList("not-existed"), groupName)); + + // Grant to a not-existed group + Assertions.assertThrows( + NoSuchGroupException.class, + () -> metalake.grantRolesToGroup(Lists.newArrayList(roleName), "not-existed")); + + // Revoke from a not-existed group + Assertions.assertThrows( + NoSuchGroupException.class, + () -> metalake.grantRolesToGroup(Lists.newArrayList(roleName), "not-existed")); + + // Grant a granted role + metalake.grantRolesToGroup(Lists.newArrayList(roleName), groupName); + Assertions.assertDoesNotThrow( + () -> metalake.grantRolesToGroup(Lists.newArrayList(roleName), groupName)); + + // Revoke a revoked role + metalake.revokeRolesFromGroup(Lists.newArrayList(roleName), groupName); + Assertions.assertDoesNotThrow( + () -> metalake.revokeRolesFromGroup(Lists.newArrayList(roleName), groupName)); + + // Clean up + metalake.removeGroup(groupName); + metalake.deleteRole(roleName); + } +} diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/OwnerPostHookIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/OwnerIT.java similarity index 77% rename from integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/OwnerPostHookIT.java rename to integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/OwnerIT.java index b7c6c1788db..78411b6a34c 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/OwnerPostHookIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/OwnerIT.java @@ -33,6 +33,8 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; +import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; @@ -50,9 +52,9 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") -public class OwnerPostHookIT extends AbstractIT { +public class OwnerIT extends AbstractIT { - private static final Logger LOG = LoggerFactory.getLogger(OwnerPostHookIT.class); + private static final Logger LOG = LoggerFactory.getLogger(OwnerIT.class); private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static String hmsUri; @@ -116,12 +118,26 @@ public void testCreateFileset() { Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, owner.name()); Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Set another owner + String anotherUser = "another"; + metalake.addUser(anotherUser); + metalake.setOwner(metalakeObject, anotherUser, Owner.Type.USER); + owner = metalake.getOwner(metalakeObject).get(); + Assertions.assertEquals(anotherUser, owner.name()); + Assertions.assertEquals(Owner.Type.USER, owner.type()); + MetadataObject catalogObject = MetadataObjects.of(Lists.newArrayList(catalogNameA), MetadataObject.Type.CATALOG); owner = metalake.getOwner(catalogObject).get(); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, owner.name()); Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Set another owner + metalake.setOwner(catalogObject, anotherUser, Owner.Type.USER); + owner = metalake.getOwner(catalogObject).get(); + Assertions.assertEquals(anotherUser, owner.name()); + Assertions.assertEquals(Owner.Type.USER, owner.type()); + MetadataObject schemaObject = MetadataObjects.of( Lists.newArrayList(catalogNameA, "schema_owner"), MetadataObject.Type.SCHEMA); @@ -129,6 +145,12 @@ public void testCreateFileset() { Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, owner.name()); Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Set another owner + metalake.setOwner(schemaObject, anotherUser, Owner.Type.USER); + owner = metalake.getOwner(schemaObject).get(); + Assertions.assertEquals(anotherUser, owner.name()); + Assertions.assertEquals(Owner.Type.USER, owner.type()); + MetadataObject filesetObject = MetadataObjects.of( Lists.newArrayList(catalogNameA, "schema_owner", "fileset_owner"), @@ -137,6 +159,12 @@ public void testCreateFileset() { Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, owner.name()); Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Set another owner + metalake.setOwner(filesetObject, anotherUser, Owner.Type.USER); + owner = metalake.getOwner(catalogObject).get(); + Assertions.assertEquals(anotherUser, owner.name()); + Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Clean up catalog.asFilesetCatalog().dropFileset(fileIdent); catalog.asSchemas().dropSchema("schema_owner", true); @@ -181,6 +209,14 @@ public void testCreateTopic() { Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, owner.name()); Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Set another owner + String anotherUser = "another"; + metalake.addUser(anotherUser); + metalake.setOwner(topicObject, anotherUser, Owner.Type.USER); + owner = metalake.getOwner(topicObject).get(); + Assertions.assertEquals(anotherUser, owner.name()); + Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Clean up catalogB.asTopicCatalog().dropTopic(topicIdent); metalake.dropCatalog(catalogNameB); @@ -209,6 +245,14 @@ public void testCreateRole() { Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, owner.name()); Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Set another owner + String anotherUser = "another"; + metalake.addUser(anotherUser); + metalake.setOwner(roleObject, anotherUser, Owner.Type.USER); + owner = metalake.getOwner(roleObject).get(); + Assertions.assertEquals(anotherUser, owner.name()); + Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Clean up metalake.deleteRole("role_owner"); client.dropMetalake(metalakeNameC); @@ -265,10 +309,49 @@ public void testCreateTable() { Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, owner.name()); Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Set another owner + String anotherUser = "another"; + metalake.addUser(anotherUser); + metalake.setOwner(tableObject, anotherUser, Owner.Type.USER); + owner = metalake.getOwner(tableObject).get(); + Assertions.assertEquals(anotherUser, owner.name()); + Assertions.assertEquals(Owner.Type.USER, owner.type()); + // Clean up catalog.asTableCatalog().dropTable(tableIdent); catalog.asSchemas().dropSchema("schema_owner", true); metalake.dropCatalog(catalogNameD); client.dropMetalake(metalakeNameD); } + + @Test + public void testOwnerWithException() { + String metalakeNameE = RandomNameUtils.genRandomName("metalakeE"); + String catalogNameE = RandomNameUtils.genRandomName("catalogE"); + GravitinoMetalake metalake = + client.createMetalake(metalakeNameE, "metalake E comment", Collections.emptyMap()); + + MetadataObject metalakeObject = + MetadataObjects.of(Lists.newArrayList(metalakeNameE), MetadataObject.Type.METALAKE); + + MetadataObject catalogObject = + MetadataObjects.of(Lists.newArrayList(catalogNameE), MetadataObject.Type.CATALOG); + + // Get a not-existed catalog + Assertions.assertThrows( + NoSuchMetadataObjectException.class, () -> metalake.getOwner(catalogObject)); + + // Set a not-existed catalog + Assertions.assertThrows( + NotFoundException.class, + () -> metalake.setOwner(catalogObject, AuthConstants.ANONYMOUS_USER, Owner.Type.USER)); + + // Set a not-existed user + Assertions.assertThrows( + NotFoundException.class, + () -> metalake.setOwner(metalakeObject, "not-existed", Owner.Type.USER)); + + // Cleanup + client.dropMetalake(metalakeNameE); + } } diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/AbstractRangerIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerIT.java rename to integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/AbstractRangerIT.java index 90cc1b14ffa..80c0e49608b 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/AbstractRangerIT.java @@ -36,8 +36,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class RangerIT { - private static final Logger LOG = LoggerFactory.getLogger(RangerIT.class); +public abstract class AbstractRangerIT { + private static final Logger LOG = LoggerFactory.getLogger(AbstractRangerIT.class); protected static final String RANGER_TRINO_REPO_NAME = "trinoDev"; private static final String RANGER_TRINO_TYPE = "trino"; protected static final String RANGER_HIVE_REPO_NAME = "hiveDev"; diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java index db861699eeb..1f011627296 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/authorization/ranger/RangerHiveIT.java @@ -40,7 +40,7 @@ import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") -public class RangerHiveIT extends RangerIT { +public class RangerHiveIT extends AbstractRangerIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static Connection adminConnection; private static Connection anonymousConnection; @@ -49,7 +49,7 @@ public class RangerHiveIT extends RangerIT { @BeforeAll public static void setup() { - RangerIT.setup(); + AbstractRangerIT.setup(); containerSuite.startHiveRangerContainer( new HashMap<>( @@ -62,9 +62,9 @@ public static void setup() { containerSuite.getRangerContainer().getContainerIpAddress(), RangerContainer.RANGER_SERVER_PORT), RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME, - RangerIT.RANGER_HIVE_REPO_NAME, + AbstractRangerIT.RANGER_HIVE_REPO_NAME, RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME, - RangerIT.RANGER_HDFS_REPO_NAME, + AbstractRangerIT.RANGER_HDFS_REPO_NAME, HiveContainer.HADOOP_USER_NAME, adminUser))); diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/AuditIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/AuditIT.java index b635dc1f986..c5135b8fef5 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/AuditIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/AuditIT.java @@ -30,10 +30,8 @@ import org.apache.gravitino.utils.RandomNameUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-test") public class AuditIT extends AbstractIT { private static final String expectUser = System.getProperty("user.name"); @@ -47,7 +45,7 @@ public static void startIntegrationTest() throws Exception { } @Test - public void testAuditMetalake() throws Exception { + public void testAuditMetalake() { String metalakeAuditName = RandomNameUtils.genRandomName("metalakeAudit"); String newName = RandomNameUtils.genRandomName("newmetaname"); diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/MetalakeIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/MetalakeIT.java index 3d87e9d4c28..bbd93ff2e69 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/MetalakeIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/MetalakeIT.java @@ -39,11 +39,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; -@Tag("gravitino-docker-test") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class MetalakeIT extends AbstractIT { public static String metalakeNameA = RandomNameUtils.genRandomName("metalakeA"); diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/KerberosOperationsIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/KerberosOperationsIT.java index 16de7b06e1e..668922352df 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/KerberosOperationsIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/KerberosOperationsIT.java @@ -40,11 +40,9 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.testcontainers.shaded.com.google.common.util.concurrent.Uninterruptibles; -@Tag("gravitino-docker-test") public class KerberosOperationsIT extends AbstractIT { private static final KerberosSecurityTestcase kdc = diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/OAuth2OperationsIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/OAuth2OperationsIT.java index e4f10769bac..af921acb59c 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/OAuth2OperationsIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/OAuth2OperationsIT.java @@ -36,10 +36,8 @@ import org.apache.gravitino.server.authentication.OAuthConfig; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-test") public class OAuth2OperationsIT extends AbstractIT { private static final KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/VersionOperationsIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/VersionOperationsIT.java index 59ba215f9e7..40c87560171 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/VersionOperationsIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/rest/VersionOperationsIT.java @@ -22,10 +22,8 @@ import org.apache.gravitino.integration.test.util.AbstractIT; import org.apache.gravitino.integration.test.util.ITUtils; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-test") public class VersionOperationsIT extends AbstractIT { @Test public void testGetVersion() { diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/MetalakePageTest.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/MetalakePageTest.java index e8894975344..6a87c2f6836 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/MetalakePageTest.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/web/ui/MetalakePageTest.java @@ -23,11 +23,11 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.condition.DisabledIfSystemProperty; -@Tag("gravitino-docker-test") +@DisabledIfSystemProperty(named = "testMode", matches = "embedded") @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class MetalakePageTest extends AbstractWebIT { private static final String WEB_TITLE = "Gravitino"; diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt index 35dab129f61..be0a71f3c77 100644 --- a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt +++ b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt @@ -1,17 +1,17 @@ CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino""}" CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""test_key"":""test_value"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""test_key"":""test_value""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""test_key"":""test_value""}" CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""jdbc-password"":""ds123"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" +"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" CALL diff --git a/integration-test/trino-it/docker-compose.yaml b/integration-test/trino-it/docker-compose.yaml index 04f9264edb0..c00c1a45d83 100644 --- a/integration-test/trino-it/docker-compose.yaml +++ b/integration-test/trino-it/docker-compose.yaml @@ -16,12 +16,10 @@ # specific language governing permissions and limitations # under the License. -# -version: '3.0' services: hive: - image: datastrato/gravitino-ci-hive:0.1.12 + image: datastrato/gravitino-ci-hive:0.1.13 networks: - trino-net container_name: trino-ci-hive @@ -30,8 +28,6 @@ services: entrypoint: /bin/bash /tmp/hive/init.sh volumes: - ./init/hive:/tmp/hive - - ../build/trino-ci-container-log/hive:/tmp/root - - ../build/trino-ci-container-log/hdfs:/usr/local/hadoop/logs healthcheck: test: ["CMD", "/tmp/check-status.sh"] interval: 10s diff --git a/integration-test/trino-it/init/hive/init.sh b/integration-test/trino-it/init/hive/init.sh index b909fff8658..35d9a8ccfca 100755 --- a/integration-test/trino-it/init/hive/init.sh +++ b/integration-test/trino-it/init/hive/init.sh @@ -16,9 +16,7 @@ # specific language governing permissions and limitations # under the License. -# - IP=$(hostname -I | awk '{print $1}') -sed -i "s|hdfs://localhost:9000|hdfs://${IP}:9000|g" /usr/local/hive/conf/hive-site.xml +sed -i "s|hdfs://__REPLACE__HOST_NAME:9000|hdfs://${IP}:9000|g" ${HIVE_TMP_CONF_DIR}/hive-site.xml /bin/bash /usr/local/sbin/start.sh diff --git a/integration-test/trino-it/launch.sh b/integration-test/trino-it/launch.sh index 469c9abf707..8705d8c02a6 100755 --- a/integration-test/trino-it/launch.sh +++ b/integration-test/trino-it/launch.sh @@ -70,8 +70,3 @@ done echo "All docker compose service is now available." - -# change the hive container's logs directory permission -docker exec trino-ci-hive chown -R `id -u`:`id -g` /tmp/root -docker exec trino-ci-hive chown -R `id -u`:`id -g` /usr/local/hadoop/logs - diff --git a/integration-test/trino-it/shutdown.sh b/integration-test/trino-it/shutdown.sh index 94c17c6236e..19955472879 100755 --- a/integration-test/trino-it/shutdown.sh +++ b/integration-test/trino-it/shutdown.sh @@ -20,13 +20,8 @@ # cd "$(dirname "$0")" -# change the hive container's logs directory permission -docker exec trino-ci-hive chown -R `id -u`:`id -g` /tmp/root -docker exec trino-ci-hive chown -R `id -u`:`id -g` /usr/local/hadoop/logs - -# for trace file permission -ls -l ../build/trino-ci-container-log -ls -l ../build/trino-ci-container-log/hive -ls -l ../build/trino-ci-container-log/hdfs +LOG_DIR=../build/trino-ci-container-log +docker cp trino-ci-hive:/usr/local/hadoop/logs $LOG_DIR/hdfs +docker cp trino-ci-hive:/tmp/root $LOG_DIR/hive docker compose down diff --git a/server-common/src/main/java/org/apache/gravitino/server/authentication/AuthenticationFilter.java b/server-common/src/main/java/org/apache/gravitino/server/authentication/AuthenticationFilter.java index 3479503d494..8e376738f7a 100644 --- a/server-common/src/main/java/org/apache/gravitino/server/authentication/AuthenticationFilter.java +++ b/server-common/src/main/java/org/apache/gravitino/server/authentication/AuthenticationFilter.java @@ -68,6 +68,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha authData = headerData.nextElement().getBytes(StandardCharsets.UTF_8); } + // If token is supported by multiple authenticators, use the first by default. Principal principal = null; for (Authenticator authenticator : authenticators) { if (authenticator.supportsToken(authData) && authenticator.isDataFromToken()) { diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java index f3e8e2f86db..a3699c0f9dc 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java @@ -46,6 +46,7 @@ import org.apache.gravitino.dto.responses.DeleteResponse; import org.apache.gravitino.dto.responses.RoleResponse; import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.lock.LockType; import org.apache.gravitino.lock.TreeLockUtils; import org.apache.gravitino.metrics.MetricNames; @@ -179,7 +180,7 @@ static void checkSecurableObject(String metalake, SecurableObjectDTO object) { identifier = NameIdentifier.parse(String.format("%s.%s", metalake, object.fullName())); } - String existErrMsg = "Securable object % doesn't exist"; + String existErrMsg = "Securable object %s doesn't exist"; TreeLockUtils.doWithTreeLock( identifier, @@ -188,41 +189,41 @@ static void checkSecurableObject(String metalake, SecurableObjectDTO object) { switch (object.type()) { case METALAKE: if (!GravitinoEnv.getInstance().metalakeDispatcher().metalakeExists(identifier)) { - throw new IllegalArgumentException(String.format(existErrMsg, object.fullName())); + throw new NoSuchMetadataObjectException(existErrMsg, object.fullName()); } break; case CATALOG: if (!GravitinoEnv.getInstance().catalogDispatcher().catalogExists(identifier)) { - throw new IllegalArgumentException(String.format(existErrMsg, object.fullName())); + throw new NoSuchMetadataObjectException(existErrMsg, object.fullName()); } break; case SCHEMA: if (!GravitinoEnv.getInstance().schemaDispatcher().schemaExists(identifier)) { - throw new IllegalArgumentException(String.format(existErrMsg, object.fullName())); + throw new NoSuchMetadataObjectException(existErrMsg, object.fullName()); } break; case FILESET: if (!GravitinoEnv.getInstance().filesetDispatcher().filesetExists(identifier)) { - throw new IllegalArgumentException(String.format(existErrMsg, object.fullName())); + throw new NoSuchMetadataObjectException(existErrMsg, object.fullName()); } break; case TABLE: if (!GravitinoEnv.getInstance().tableDispatcher().tableExists(identifier)) { - throw new IllegalArgumentException(String.format(existErrMsg, object.fullName())); + throw new NoSuchMetadataObjectException(existErrMsg, object.fullName()); } break; case TOPIC: if (!GravitinoEnv.getInstance().topicDispatcher().topicExists(identifier)) { - throw new IllegalArgumentException(String.format(existErrMsg, object.fullName())); + throw new NoSuchMetadataObjectException(existErrMsg, object.fullName()); } break; diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java index 6c34547d07d..a00f91b4d03 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java @@ -56,6 +56,7 @@ import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.dto.responses.RoleResponse; import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; @@ -199,10 +200,10 @@ public void testCreateRole() { .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); - Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), respNotExist.getStatus()); + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), respNotExist.getStatus()); Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, respNotExist.getMediaType()); ErrorResponse notExistResponse = respNotExist.readEntity(ErrorResponse.class); - Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, notExistResponse.getCode()); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, notExistResponse.getCode()); // Test to throw NoSuchMetalakeException when(catalogDispatcher.catalogExists(any())).thenReturn(true); @@ -402,7 +403,7 @@ public void testCheckSecurableObjects() { () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(catalog))); when(catalogDispatcher.catalogExists(any())).thenReturn(false); Assertions.assertThrows( - IllegalArgumentException.class, + NoSuchMetadataObjectException.class, () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(catalog))); // check the schema @@ -414,7 +415,7 @@ public void testCheckSecurableObjects() { () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(schema))); when(schemaDispatcher.schemaExists(any())).thenReturn(false); Assertions.assertThrows( - IllegalArgumentException.class, + NoSuchMetadataObjectException.class, () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(schema))); // check the table @@ -426,7 +427,7 @@ public void testCheckSecurableObjects() { () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(table))); when(tableDispatcher.tableExists(any())).thenReturn(false); Assertions.assertThrows( - IllegalArgumentException.class, + NoSuchMetadataObjectException.class, () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(table))); // check the topic @@ -438,7 +439,7 @@ public void testCheckSecurableObjects() { () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(topic))); when(topicDispatcher.topicExists(any())).thenReturn(false); Assertions.assertThrows( - IllegalArgumentException.class, + NoSuchMetadataObjectException.class, () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(topic))); // check the fileset @@ -450,7 +451,7 @@ public void testCheckSecurableObjects() { () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(fileset))); when(filesetDispatcher.filesetExists(any())).thenReturn(false); Assertions.assertThrows( - IllegalArgumentException.class, + NoSuchMetadataObjectException.class, () -> RoleOperations.checkSecurableObject("metalake", DTOConverters.toDTO(fileset))); } } diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/extensions/IcebergExtendedDataSourceV2Strategy.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/extensions/IcebergExtendedDataSourceV2Strategy.java index 9132ac3c493..3bf2631569f 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/extensions/IcebergExtendedDataSourceV2Strategy.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/extensions/IcebergExtendedDataSourceV2Strategy.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.spark.connector.iceberg.extensions; +import static org.apache.gravitino.spark.connector.utils.ConnectorUtil.toJavaList; + import java.util.Collections; import org.apache.gravitino.spark.connector.iceberg.GravitinoIcebergCatalog; import org.apache.iceberg.spark.Spark3Util; @@ -51,7 +53,7 @@ import scala.Option; import scala.Some; import scala.collection.JavaConverters; -import scala.collection.Seq; +import scala.collection.immutable.Seq; public class IcebergExtendedDataSourceV2Strategy extends ExtendedDataSourceV2Strategy { @@ -66,7 +68,8 @@ public IcebergExtendedDataSourceV2Strategy(SparkSession spark) { public Seq apply(LogicalPlan plan) { if (plan instanceof AddPartitionField) { AddPartitionField addPartitionField = (AddPartitionField) plan; - return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier(spark, addPartitionField.table()) + return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( + spark, addPartitionField.table().toIndexedSeq()) .map( catalogAndIdentifier -> { AddPartitionFieldExec addPartitionFieldExec = @@ -81,7 +84,7 @@ public Seq apply(LogicalPlan plan) { } else if (plan instanceof CreateOrReplaceBranch) { CreateOrReplaceBranch createOrReplaceBranch = (CreateOrReplaceBranch) plan; return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( - spark, createOrReplaceBranch.table()) + spark, createOrReplaceBranch.table().toIndexedSeq()) .map( catalogAndIdentifier -> { CreateOrReplaceBranchExec createOrReplaceBranchExec = @@ -99,7 +102,7 @@ public Seq apply(LogicalPlan plan) { } else if (plan instanceof CreateOrReplaceTag) { CreateOrReplaceTag createOrReplaceTag = (CreateOrReplaceTag) plan; return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( - spark, createOrReplaceTag.table()) + spark, createOrReplaceTag.table().toIndexedSeq()) .map( catalogAndIdentifier -> { CreateOrReplaceTagExec createOrReplaceTagExec = @@ -116,7 +119,8 @@ public Seq apply(LogicalPlan plan) { .get(); } else if (plan instanceof DropBranch) { DropBranch dropBranch = (DropBranch) plan; - return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier(spark, dropBranch.table()) + return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( + spark, dropBranch.table().toIndexedSeq()) .map( catalogAndIdentifier -> { DropBranchExec dropBranchExec = @@ -130,7 +134,8 @@ public Seq apply(LogicalPlan plan) { .get(); } else if (plan instanceof DropTag) { DropTag dropTag = (DropTag) plan; - return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier(spark, dropTag.table()) + return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( + spark, dropTag.table().toIndexedSeq()) .map( catalogAndIdentifier -> { DropTagExec dropTagExec = @@ -145,7 +150,7 @@ public Seq apply(LogicalPlan plan) { } else if (plan instanceof DropPartitionField) { DropPartitionField dropPartitionField = (DropPartitionField) plan; return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( - spark, dropPartitionField.table()) + spark, dropPartitionField.table().toIndexedSeq()) .map( catalogAndIdentifier -> { DropPartitionFieldExec dropPartitionFieldExec = @@ -159,7 +164,7 @@ public Seq apply(LogicalPlan plan) { } else if (plan instanceof ReplacePartitionField) { ReplacePartitionField replacePartitionField = (ReplacePartitionField) plan; return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( - spark, replacePartitionField.table()) + spark, replacePartitionField.table().toIndexedSeq()) .map( catalogAndIdentifier -> { ReplacePartitionFieldExec replacePartitionFieldExec = @@ -175,7 +180,7 @@ public Seq apply(LogicalPlan plan) { } else if (plan instanceof SetIdentifierFields) { SetIdentifierFields setIdentifierFields = (SetIdentifierFields) plan; return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( - spark, setIdentifierFields.table()) + spark, setIdentifierFields.table().toIndexedSeq()) .map( catalogAndIdentifier -> { SetIdentifierFieldsExec setIdentifierFieldsExec = @@ -189,7 +194,7 @@ public Seq apply(LogicalPlan plan) { } else if (plan instanceof DropIdentifierFields) { DropIdentifierFields dropIdentifierFields = (DropIdentifierFields) plan; return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( - spark, dropIdentifierFields.table()) + spark, dropIdentifierFields.table().toIndexedSeq()) .map( catalogAndIdentifier -> { DropIdentifierFieldsExec dropIdentifierFieldsExec = @@ -204,7 +209,7 @@ public Seq apply(LogicalPlan plan) { SetWriteDistributionAndOrdering setWriteDistributionAndOrdering = (SetWriteDistributionAndOrdering) plan; return IcebergCatalogAndIdentifier.buildCatalogAndIdentifier( - spark, setWriteDistributionAndOrdering.table()) + spark, setWriteDistributionAndOrdering.table().toIndexedSeq()) .map( catalogAndIdentifier -> { SetWriteDistributionAndOrderingExec setWriteDistributionAndOrderingExec = @@ -217,14 +222,18 @@ public Seq apply(LogicalPlan plan) { }) .get(); } else { - return super.apply(plan); + scala.collection.Seq sparkPlans = super.apply(plan); + if (sparkPlans != null) { + return sparkPlans.toIndexedSeq(); + } + return null; } } private Seq toSeq(SparkPlan plan) { return JavaConverters.asScalaIteratorConverter(Collections.singletonList(plan).listIterator()) .asScala() - .toSeq(); + .toIndexedSeq(); } static class IcebergCatalogAndIdentifier { @@ -244,7 +253,7 @@ private static IcebergCatalogAndIdentifier of(TableCatalog catalog, Identifier i static Option buildCatalogAndIdentifier( SparkSession spark, Seq identifiers) { Spark3Util.CatalogAndIdentifier catalogAndIdentifier = - Spark3Util.catalogAndIdentifier(spark, JavaConverters.seqAsJavaList(identifiers)); + Spark3Util.catalogAndIdentifier(spark, toJavaList(identifiers)); CatalogPlugin catalog = catalogAndIdentifier.catalog(); if (catalog instanceof GravitinoIcebergCatalog) { return new Some<>( diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/ConnectorUtil.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/ConnectorUtil.java index 55033315551..672369854a0 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/ConnectorUtil.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/utils/ConnectorUtil.java @@ -19,11 +19,14 @@ package org.apache.gravitino.spark.connector.utils; +import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.spark.connector.ConnectorConstants; +import scala.collection.immutable.Seq; public class ConnectorUtil { @@ -37,4 +40,13 @@ public static String removeDuplicateSparkExtensions( .reduce((element1, element2) -> element1 + ConnectorConstants.COMMA + element2) .orElse(""); } + + public static List toJavaList(Seq seq) { + List javaList = new ArrayList<>(); + if (seq == null || seq.isEmpty()) { + return javaList; + } + seq.foreach(javaList::add); + return javaList; + } } diff --git a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SQLQueryTestHelper.java b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SQLQueryTestHelper.java index d86cc686c78..39c29a4a087 100644 --- a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SQLQueryTestHelper.java +++ b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/sql/SQLQueryTestHelper.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.spark.connector.integration.test.sql; +import static org.apache.gravitino.spark.connector.utils.ConnectorUtil.toJavaList; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -31,7 +33,6 @@ import org.apache.spark.sql.execution.SQLExecution; import org.apache.spark.sql.types.StructType; import scala.Option; -import scala.collection.JavaConverters; public class SQLQueryTestHelper { private static final String notIncludedMsg = "[not included in comparison]"; @@ -62,8 +63,9 @@ public static Pair> getNormalizedResult(SparkSession sessio df.queryExecution(), Option.apply(""), () -> - JavaConverters.seqAsJavaList( - HiveResult.hiveResultString(df.queryExecution().executedPlan())) + toJavaList( + HiveResult.hiveResultString(df.queryExecution().executedPlan()) + .toIndexedSeq()) .stream() .map(s -> replaceNotIncludedMsg(s)) .filter(s -> !s.isEmpty()) diff --git a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java index 06d80f59f76..6d540d21d28 100644 --- a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java +++ b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/table/GravitinoSystemTableCatalog.java @@ -29,7 +29,11 @@ import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.SchemaTableName; +import java.util.ArrayList; import java.util.List; +import java.util.TreeMap; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.trino.connector.GravitinoErrorCode; import org.apache.gravitino.trino.connector.catalog.CatalogConnectorManager; import org.apache.gravitino.trino.connector.metadata.GravitinoCatalog; @@ -56,21 +60,36 @@ public GravitinoSystemTableCatalog(CatalogConnectorManager catalogConnectorManag @Override public Page loadPageData() { - List catalogs = catalogConnectorManager.getCatalogs(); - int size = catalogs.size(); + List gravitinoCatalogs = new ArrayList<>(); + // retrieve catalogs form the Gravitino server with the configuration metalakes, + // the catalogConnectorManager does not manager catalogs in worker nodes + catalogConnectorManager + .getUsedMetalakes() + .forEach( + (metalakeName) -> { + GravitinoMetalake metalake = catalogConnectorManager.getMetalake(metalakeName); + Catalog[] catalogs = metalake.listCatalogsInfo(); + for (Catalog catalog : catalogs) { + if (catalog.type() == Catalog.Type.RELATIONAL) { + gravitinoCatalogs.add(new GravitinoCatalog(metalakeName, catalog)); + } + } + }); + int size = gravitinoCatalogs.size(); BlockBuilder nameColumnBuilder = VARCHAR.createBlockBuilder(null, size); BlockBuilder providerColumnBuilder = VARCHAR.createBlockBuilder(null, size); BlockBuilder propertyColumnBuilder = VARCHAR.createBlockBuilder(null, size); - for (GravitinoCatalog catalog : catalogs) { + for (GravitinoCatalog catalog : gravitinoCatalogs) { Preconditions.checkNotNull(catalog, "catalog should not be null"); VARCHAR.writeString(nameColumnBuilder, catalog.getName()); VARCHAR.writeString(providerColumnBuilder, catalog.getProvider()); try { VARCHAR.writeString( - propertyColumnBuilder, new ObjectMapper().writeValueAsString(catalog.getProperties())); + propertyColumnBuilder, + new ObjectMapper().writeValueAsString(new TreeMap<>(catalog.getProperties()))); } catch (JsonProcessingException e) { throw new TrinoException( GravitinoErrorCode.GRAVITINO_ILLEGAL_ARGUMENT, "Invalid property format", e); // diff --git a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java index 0cc026f9ebc..8cfb157b1b3 100644 --- a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java @@ -201,6 +201,15 @@ public Catalog answer(InvocationOnMock invocation) throws Throwable { return metalakes.get(metalakeName).catalogs.get(catalogName); } }); + when(metaLake.listCatalogsInfo()) + .thenAnswer( + new Answer() { + @Override + public Catalog[] answer(InvocationOnMock invocation) throws Throwable { + return metalakes.get(metalakeName).catalogs.values().toArray(new Catalog[0]); + } + }); + metalakes.put(metalakeName, new Metalake(metaLake)); return metaLake; }