From 4ba4d37da886612b39d9e8bc1cf138b2fa9daa63 Mon Sep 17 00:00:00 2001 From: mchades Date: Wed, 16 Oct 2024 15:40:08 +0800 Subject: [PATCH] re-define drop metalake --- .../java/org/apache/gravitino/Metalake.java | 3 + .../apache/gravitino/SupportsMetalakes.java | 74 ++++++- .../integration/test/RangerHiveE2EIT.java | 6 +- .../integration/test/HadoopCatalogIT.java | 7 +- .../test/HadoopUserAuthenticationIT.java | 2 +- .../test/HadoopUserImpersonationIT.java | 5 +- .../hive/integration/test/CatalogHiveIT.java | 11 +- .../integration/test/CatalogDorisIT.java | 2 +- .../integration/test/AuditCatalogMysqlIT.java | 2 +- .../integration/test/CatalogMysqlIT.java | 1 + .../integration/test/CatalogPostgreSqlIT.java | 1 + .../integration/test/CatalogKafkaIT.java | 1 + .../test/CatalogIcebergBaseIT.java | 1 + .../integration/test/CatalogPaimonBaseIT.java | 1 + .../client/GravitinoAdminClient.java | 60 +++++- .../client/integration/test/AuditIT.java | 1 + .../client/integration/test/MetalakeIT.java | 5 +- .../client/integration/test/TagIT.java | 2 +- .../AccessControlNotAllowIT.java | 3 +- .../authorization/CheckCurrentUserIT.java | 2 +- .../test/authorization/OwnerIT.java | 10 +- .../client/gravitino_admin_client.py | 31 ++- .../handlers/metalake_error_handler.py | 9 + .../integration/auth/test_auth_common.py | 4 +- .../tests/integration/test_catalog.py | 4 +- .../tests/integration/test_fileset_catalog.py | 4 +- .../tests/integration/test_metalake.py | 6 +- .../tests/integration/test_schema.py | 4 +- .../test/GravitinoVirtualFileSystemIT.java | 2 +- .../authorization/AuthorizationUtils.java | 22 -- .../gravitino/authorization/RoleManager.java | 17 +- .../authorization/UserGroupManager.java | 19 +- .../gravitino/catalog/CatalogManager.java | 46 ++--- .../connector/PropertiesMetadata.java | 2 +- .../hook/MetalakeHookDispatcher.java | 17 +- .../listener/MetalakeEventDispatcher.java | 19 +- .../apache/gravitino/meta/BaseMetalake.java | 16 +- .../gravitino/metalake/MetalakeManager.java | 194 +++++++++++++++--- .../metalake/MetalakeNormalizeDispatcher.java | 73 ++++++- .../metalake/MetalakePropertiesMetadata.java | 50 +++++ .../gravitino/metalake/SupportsMetalakes.java | 65 +++++- .../gravitino/proto/BaseMetalakeSerDe.java | 3 +- .../org/apache/gravitino/tag/TagManager.java | 28 +-- .../listener/api/event/TestMetalakeEvent.java | 3 +- .../metalake/TestMetalakeManager.java | 1 + .../gravitino/storage/TestEntityStorage.java | 10 +- gradle.properties | 2 +- .../server/web/rest/ExceptionHandlers.java | 6 + .../server/web/rest/MetalakeOperations.java | 67 +++++- .../web/rest/TestMetalakeOperations.java | 9 +- .../integration/test/TrinoQueryITBase.java | 2 +- 51 files changed, 739 insertions(+), 196 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java diff --git a/api/src/main/java/org/apache/gravitino/Metalake.java b/api/src/main/java/org/apache/gravitino/Metalake.java index fb6fdbee094..4f95a8a2365 100644 --- a/api/src/main/java/org/apache/gravitino/Metalake.java +++ b/api/src/main/java/org/apache/gravitino/Metalake.java @@ -29,6 +29,9 @@ @Evolving public interface Metalake extends Auditable { + /** The property indicating the metalake is in use. */ + String PROPERTY_IN_USE = "in-use"; + /** * The name of the metalake. * diff --git a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java index 47cce7d3a4a..93e577a7cfb 100644 --- a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java +++ b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java @@ -20,8 +20,10 @@ import java.util.Map; import org.apache.gravitino.annotation.Evolving; +import org.apache.gravitino.exceptions.EntityInUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Client interface for supporting metalakes. It includes methods for listing, loading, creating, @@ -38,7 +40,7 @@ public interface SupportsMetalakes { Metalake[] listMetalakes(); /** - * Load a metalake by its identifier. + * Load a metalake by its name. * * @param name the name of the metalake. * @return The metalake. @@ -62,7 +64,7 @@ default boolean metalakeExists(String name) { } /** - * Create a metalake with specified identifier. + * Create a metalake with specified name, comment and properties. * * @param name The name of the metalake. * @param comment The comment of the metalake. @@ -74,7 +76,7 @@ Metalake createMetalake(String name, String comment, Map propert throws MetalakeAlreadyExistsException; /** - * Alter a metalake with specified identifier. + * Alter a metalake with specified metalake name and changes. * * @param name The name of the metalake. * @param changes The changes to apply. @@ -86,10 +88,70 @@ Metalake alterMetalake(String name, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException; /** - * Drop a metalake with specified identifier. + * Drop a metalake with specified name. Please make sure: * - * @param name The identifier of the metalake. + * + * + * It is equivalent to calling {@code dropMetalake(ident, false)}. + * + * @param name The name of the metalake. * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty. + * @throws EntityInUseException If the metalake is in use. + */ + default boolean dropMetalake(String name) throws NonEmptyEntityException, EntityInUseException { + return dropMetalake(name, false); + } + + /** + * Drop a metalake with specified name. If the force flag is true, it will: + * + * + * + * If the force flag is false, it is equivalent to calling {@link #dropMetalake(String)}. + * + * @param name The name of the metalake. + * @param force Whether to force the drop. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws EntityInUseException If the metalake is in use and force is false. + */ + boolean dropMetalake(String name, boolean force) + throws NonEmptyEntityException, EntityInUseException; + + /** + * Activate a metalake. If the metalake is already active, this method does nothing. + * + * @param name The name of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. + */ + void activateMetalake(String name) throws NoSuchMetalakeException; + + /** + * Deactivate a metalake. If the metalake is already inactive, this method does nothing. Once a + * metalake is deactivated: + * + * + * + * @param name The name of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. */ - boolean dropMetalake(String name); + void deactivateMetalake(String name) throws NoSuchMetalakeException; } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index 0f1f4c05abd..e09d1ab246f 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -191,6 +191,7 @@ public void stop() throws IOException { })); Arrays.stream(metalake.listCatalogs()) .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); + client.deactivateMetalake(metalakeName); client.dropMetalake(metalakeName); } if (sparkSession != null) { @@ -269,10 +270,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index adf0c2ed578..0ea347d5b29 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -90,7 +90,7 @@ public void stop() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (hdfs != null) { hdfs.close(); } @@ -106,10 +106,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java index d0de2972742..d074709bd80 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java @@ -654,6 +654,6 @@ void testUserImpersonation() { catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, filesetName)); catalog.asSchemas().dropSchema(SCHEMA_NAME, true); gravitinoMetalake.dropCatalog(catalogName, true); - adminClient.dropMetalake(metalakeName); + adminClient.dropMetalake(metalakeName, true); } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java index 9515b45b5dd..808aafbaef5 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java @@ -258,10 +258,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index 31493d54ba2..903f3fe8a41 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -229,7 +229,7 @@ public void stop() throws IOException { })); Arrays.stream(metalake.listCatalogs()) .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); } if (hiveClientPool != null) { hiveClientPool.close(); @@ -264,10 +264,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } @@ -1429,8 +1428,8 @@ void testDropAndRename() { client.createMetalake(metalakeName1, "comment", Collections.emptyMap()); client.createMetalake(metalakeName2, "comment", Collections.emptyMap()); - client.dropMetalake(metalakeName1); - client.dropMetalake(metalakeName2); + client.dropMetalake(metalakeName1, true); + client.dropMetalake(metalakeName2, true); client.createMetalake(metalakeName1, "comment", Collections.emptyMap()); diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index b08aa491600..d5b0fde0afa 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -126,7 +126,7 @@ public void startup() throws IOException { public void stop() { clearTableAndSchema(); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); } @AfterEach diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java index 784361d6407..5451e41db5b 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java @@ -77,7 +77,7 @@ public void startIntegrationTest() throws Exception { @AfterAll public void stopIntegrationTest() throws IOException, InterruptedException { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); mysqlService.close(); super.stopIntegrationTest(); } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index 33408e28bfd..59cb66d2f00 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -144,6 +144,7 @@ public void stop() { clearTableAndSchema(); metalake.deactivateCatalog(catalogName); metalake.dropCatalog(catalogName); + client.deactivateMetalake(metalakeName); client.dropMetalake(metalakeName); mysqlService.close(); } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index d9b30fb5112..f29f15abf86 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -130,6 +130,7 @@ public void stop() { } metalake.deactivateCatalog(catalogName); metalake.dropCatalog(catalogName); + client.deactivateMetalake(metalakeName); client.dropMetalake(metalakeName); postgreSqlService.close(); } diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index e6e388f21a3..9f68b671d06 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -126,6 +126,7 @@ public void shutdown() { metalake.deactivateCatalog(catalogName); metalake.dropCatalog(catalogName); })); + client.deactivateMetalake(METALAKE_NAME); client.dropMetalake(METALAKE_NAME); if (adminClient != null) { adminClient.close(); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index 35755b70758..bf1d236df31 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -140,6 +140,7 @@ public void stop() throws Exception { clearTableAndSchema(); metalake.deactivateCatalog(catalogName); metalake.dropCatalog(catalogName); + client.deactivateMetalake(metalakeName); client.dropMetalake(metalakeName); } finally { if (spark != null) { diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java index 61778da91e7..81c1cc988a0 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java @@ -138,6 +138,7 @@ public void stop() { clearTableAndSchema(); metalake.deactivateCatalog(catalogName); metalake.dropCatalog(catalogName); + client.deactivateMetalake(metalakeName); client.dropMetalake(metalakeName); if (spark != null) { spark.close(); diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java index 936edcf3c2a..693cbb7fbaf 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -31,10 +32,13 @@ import org.apache.gravitino.dto.requests.MetalakeUpdateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdatesRequest; import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.dto.responses.MetalakeListResponse; import org.apache.gravitino.dto.responses.MetalakeResponse; +import org.apache.gravitino.exceptions.EntityInUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Apache Gravitino Client for the administrator to interact with the Gravitino API, allowing the @@ -145,17 +149,33 @@ public GravitinoMetalake alterMetalake(String name, MetalakeChange... changes) } /** - * Drops a specific Metalake using the Gravitino API. + * Drop a metalake with specified name. Please make sure: * - * @param name The name of the Metalake to be dropped. - * @return True if the Metalake was successfully dropped, false if the Metalake does not exist. + * + * + * It is equivalent to calling {@code dropMetalake(ident, false)}. + * + * @param name The name of the metalake. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty. + * @throws EntityInUseException If the metalake is in use. */ @Override - public boolean dropMetalake(String name) { + public boolean dropMetalake(String name, boolean force) + throws NonEmptyEntityException, EntityInUseException { checkMetalakeName(name); + Map params = new HashMap<>(); + params.put("force", String.valueOf(force)); + DropResponse resp = restClient.delete( API_METALAKES_IDENTIFIER_PATH + name, + params, DropResponse.class, Collections.emptyMap(), ErrorHandlers.metalakeErrorHandler()); @@ -163,6 +183,38 @@ public boolean dropMetalake(String name) { return resp.dropped(); } + @Override + public void activateMetalake(String name) throws NoSuchMetalakeException { + ErrorResponse resp = + restClient.get( + API_METALAKES_IDENTIFIER_PATH + name + "/activate", + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.metalakeErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.metalakeErrorHandler().accept(resp); + } + + @Override + public void deactivateMetalake(String name) throws NoSuchMetalakeException { + ErrorResponse resp = + restClient.get( + API_METALAKES_IDENTIFIER_PATH + name + "/deactivate", + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.metalakeErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.metalakeErrorHandler().accept(resp); + } + /** * Creates a new builder for constructing a GravitinoClient. * diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java index c438e0fca1e..6483ac60f23 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java @@ -60,6 +60,7 @@ public void testAuditMetalake() { metaLake = client.alterMetalake(metalakeAuditName, changes); Assertions.assertEquals(expectUser, metaLake.auditInfo().creator()); Assertions.assertEquals(expectUser, metaLake.auditInfo().lastModifier()); + Assertions.assertDoesNotThrow(() -> client.deactivateMetalake(newName)); Assertions.assertTrue(client.dropMetalake(newName), "metaLake should be dropped"); Assertions.assertFalse(client.dropMetalake(newName), "metalake should be non-existent"); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java index fb9efd2ca7f..4130e5ebe57 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.client.integration.test; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -182,6 +183,7 @@ public void testCreateMetalakeWithChinese() { public void testDropMetalakes() { GravitinoMetalake metalakeA = client.createMetalake(metalakeNameA, "metalake A comment", Collections.emptyMap()); + assertDoesNotThrow(() -> client.deactivateMetalake(metalakeA.name())); assertTrue(client.dropMetalake(metalakeA.name()), "metaLake should be dropped"); NameIdentifier id = NameIdentifier.of(metalakeNameA); assertThrows( @@ -205,12 +207,13 @@ public void testUpdateMetalakeWithNullableComment() { new MetalakeChange[] {MetalakeChange.updateComment("new metalake comment")}; GravitinoMetalake updatedMetalake = client.alterMetalake(metalakeNameA, changes); assertEquals("new metalake comment", updatedMetalake.comment()); - client.dropMetalake(metalakeNameA); + assertTrue(client.dropMetalake(metalakeNameA, true)); } public void dropMetalakes() { GravitinoMetalake[] metaLakes = client.listMetalakes(); for (GravitinoMetalake metalake : metaLakes) { + assertDoesNotThrow(() -> client.deactivateMetalake(metalake.name())); assertTrue(client.dropMetalake(metalake.name())); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java index f5b5e1dab56..dc82dfd67df 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java @@ -112,7 +112,7 @@ public void tearDown() { relationalCatalog.asTableCatalog().dropTable(NameIdentifier.of(schema.name(), table.name())); relationalCatalog.asSchemas().dropSchema(schema.name(), true); metalake.dropCatalog(relationalCatalog.name(), true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java index a6817b27418..05bbc94c543 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.integration.test.authorization; +package org.apache.gravitino.client.integration.test.authorization; import com.google.common.collect.Lists; import java.util.Collections; @@ -148,6 +148,7 @@ public void testNotAllowFilter() { Assertions.assertTrue( e.getMessage().contains("You should set 'gravitino.authorization.enable'")); + client.deactivateMetalake(metalakeTestName); client.dropMetalake(metalakeTestName); } } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java index f5615beae06..a7339ba0db1 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java @@ -98,7 +98,7 @@ public void startIntegrationTest() throws Exception { @AfterAll public void tearDown() { if (client != null) { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); client.close(); client = null; } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index 99f1e830692..daac8002ab9 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -169,7 +169,7 @@ public void testCreateFileset() { catalog.asFilesetCatalog().dropFileset(fileIdent); catalog.asSchemas().dropSchema("schema_owner", true); metalake.dropCatalog(catalogNameA, true); - client.dropMetalake(metalakeNameA); + client.dropMetalake(metalakeNameA, true); } @Test @@ -220,7 +220,7 @@ public void testCreateTopic() { // Clean up catalogB.asTopicCatalog().dropTopic(topicIdent); metalake.dropCatalog(catalogNameB, true); - client.dropMetalake(metalakeNameB); + client.dropMetalake(metalakeNameB, true); } @Test @@ -255,7 +255,7 @@ public void testCreateRole() { // Clean up metalake.deleteRole("role_owner"); - client.dropMetalake(metalakeNameC); + client.dropMetalake(metalakeNameC, true); } @Test @@ -321,7 +321,7 @@ public void testCreateTable() { catalog.asTableCatalog().dropTable(tableIdent); catalog.asSchemas().dropSchema("schema_owner", true); metalake.dropCatalog(catalogNameD, true); - client.dropMetalake(metalakeNameD); + client.dropMetalake(metalakeNameD, true); } @Test @@ -352,6 +352,6 @@ public void testOwnerWithException() { () -> metalake.setOwner(metalakeObject, "not-existed", Owner.Type.USER)); // Cleanup - client.dropMetalake(metalakeNameE); + client.dropMetalake(metalakeNameE, true); } } diff --git a/clients/client-python/gravitino/client/gravitino_admin_client.py b/clients/client-python/gravitino/client/gravitino_admin_client.py index b730a765b21..22411566e05 100644 --- a/clients/client-python/gravitino/client/gravitino_admin_client.py +++ b/clients/client-python/gravitino/client/gravitino_admin_client.py @@ -112,20 +112,47 @@ def alter_metalake(self, name: str, *changes: MetalakeChange) -> GravitinoMetala return GravitinoMetalake(metalake, self._rest_client) - def drop_metalake(self, name: str) -> bool: + def drop_metalake(self, name: str, force: bool = False) -> bool: """Drops a specific Metalake using the Gravitino API. Args: name: The name of the Metalake to be dropped. + force: Whether to force the drop operation. Returns: - True if the Metalake was successfully dropped, false otherwise. + True if the Metalake was successfully dropped, false if the Metalake does not exist. """ + params = {"force": str(force)} resp = self._rest_client.delete( self.API_METALAKES_IDENTIFIER_PATH + name, + params=params, error_handler=METALAKE_ERROR_HANDLER, ) drop_response = DropResponse.from_json(resp.body, infer_missing=True) return drop_response.dropped() + + def activate_metalake(self, name: str): + """Activate the metalake with specified name. If the metalake is already activated, this method does nothing. + + Args: + name: the name of the metalake. + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + """ + url = self.API_METALAKES_IDENTIFIER_PATH + name + "/activate" + self._rest_client.get(url, error_handler=METALAKE_ERROR_HANDLER) + + def deactivate_metalake(self, name: str): + """Deactivate the metalake with specified name. If the metalake is already deactivated, does nothing. + + Args: + name: the name of the metalake. + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + """ + url = self.API_METALAKES_IDENTIFIER_PATH + name + "/deactivate" + self._rest_client.get(url, error_handler=METALAKE_ERROR_HANDLER) diff --git a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py index 86dc575702e..de3041e0e0b 100644 --- a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py +++ b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py @@ -21,6 +21,8 @@ from gravitino.exceptions.base import ( NoSuchMetalakeException, MetalakeAlreadyExistsException, + EntityInUseException, + NonInUseEntityException, ) @@ -33,9 +35,16 @@ def handle(self, error_response: ErrorResponse): if code == ErrorConstants.NOT_FOUND_CODE: raise NoSuchMetalakeException(error_message) + if code == ErrorConstants.ALREADY_EXISTS_CODE: raise MetalakeAlreadyExistsException(error_message) + if code == ErrorConstants.ENTITY_IN_USE_CODE: + raise EntityInUseException(error_message) + + if code == ErrorConstants.NON_IN_USE_ENTITY_CODE: + raise NonInUseEntityException(error_message) + super().handle(error_response) diff --git a/clients/client-python/tests/integration/auth/test_auth_common.py b/clients/client-python/tests/integration/auth/test_auth_common.py index 2f1506319a7..ede3a4e2ad2 100644 --- a/clients/client-python/tests/integration/auth/test_auth_common.py +++ b/clients/client-python/tests/integration/auth/test_auth_common.py @@ -101,7 +101,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 37e8aa233c4..00a10b9c6a0 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -92,7 +92,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 2696c170a0d..754735b16e8 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -128,7 +128,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/test_metalake.py b/clients/client-python/tests/integration/test_metalake.py index 794cb894d97..75d3a06f26c 100644 --- a/clients/client-python/tests/integration/test_metalake.py +++ b/clients/client-python/tests/integration/test_metalake.py @@ -118,7 +118,7 @@ def test_alter_metalake(self): self.assertTrue(self.metalake_properties_key1 not in metalake.properties()) def drop_metalake(self, metalake_name: str) -> bool: - return self.gravitino_admin_client.drop_metalake(metalake_name) + return self.gravitino_admin_client.drop_metalake(metalake_name, True) def test_drop_metalake(self): self.create_metalake(self.metalake_name) @@ -152,7 +152,9 @@ def test_load_metalakes(self): self.assertIsNotNone(metalake) self.assertEqual(metalake.name(), self.metalake_name) self.assertEqual(metalake.comment(), self.metalake_comment) - self.assertEqual(metalake.properties(), self.metalake_properties) + self.assertEqual( + metalake.properties(), {**self.metalake_properties, "in-use": "true"} + ) self.assertEqual(metalake.audit_info().creator(), "anonymous") def test_failed_load_metalakes(self): diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index 269693dcf25..c8a6b270b84 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -128,7 +128,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java index af0da6c8725..ae001b90f6b 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java @@ -91,7 +91,7 @@ public void tearDown() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index 81447abfd3b..1d5a2acf034 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -19,14 +19,12 @@ package org.apache.gravitino.authorization; import com.google.common.collect.Sets; -import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Set; import java.util.function.Consumer; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; -import org.apache.gravitino.EntityStore; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -40,12 +38,9 @@ import org.apache.gravitino.exceptions.IllegalPrivilegeException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchUserException; import org.apache.gravitino.utils.MetadataObjectUtil; import org.apache.gravitino.utils.NameIdentifierUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /* The utilization class of authorization module*/ public class AuthorizationUtils { @@ -53,8 +48,6 @@ public class AuthorizationUtils { static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in th metalake %s"; static final String GROUP_DOES_NOT_EXIST_MSG = "Group %s does not exist in th metalake %s"; static final String ROLE_DOES_NOT_EXIST_MSG = "Role %s does not exist in th metalake %s"; - private static final Logger LOG = LoggerFactory.getLogger(AuthorizationUtils.class); - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private static final Set FILESET_PRIVILEGES = Sets.immutableEnumSet( @@ -68,21 +61,6 @@ public class AuthorizationUtils { private AuthorizationUtils() {} - static void checkMetalakeExists(String metalake) throws NoSuchMetalakeException { - try { - EntityStore store = GravitinoEnv.getInstance().entityStore(); - - NameIdentifier metalakeIdent = NameIdentifier.of(metalake); - if (!store.exists(metalakeIdent, Entity.EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } - } catch (IOException e) { - LOG.error("Failed to do storage operation", e); - throw new RuntimeException(e); - } - } - public static void checkCurrentUser(String metalake, String user) { try { AccessControlDispatcher dispatcher = GravitinoEnv.getInstance().accessControlDispatcher(); diff --git a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java index dc675fdcef5..11c24102bca 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java @@ -19,6 +19,8 @@ package org.apache.gravitino.authorization; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.collect.Sets; import java.io.IOException; import java.time.Instant; @@ -33,7 +35,6 @@ import org.apache.gravitino.SupportsRelationOperations; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.meta.AuditInfo; @@ -52,7 +53,6 @@ class RoleManager { private static final Logger LOG = LoggerFactory.getLogger(RoleManager.class); - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private final EntityStore store; private final IdGenerator idGenerator; @@ -67,7 +67,7 @@ RoleEntity createRole( Map properties, List securableObjects) throws RoleAlreadyExistsException { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); RoleEntity roleEntity = RoleEntity.builder() .withId(idGenerator.nextId()) @@ -104,7 +104,7 @@ RoleEntity createRole( RoleEntity getRole(String metalake, String role) throws NoSuchRoleException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return getRoleEntity(AuthorizationUtils.ofRole(metalake, role)); } catch (NoSuchEntityException e) { LOG.warn("Role {} does not exist in the metalake {}", role, metalake, e); @@ -114,7 +114,7 @@ RoleEntity getRole(String metalake, String role) throws NoSuchRoleException { boolean deleteRole(String metalake, String role) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); NameIdentifier ident = AuthorizationUtils.ofRole(metalake, role); try { @@ -138,14 +138,11 @@ boolean deleteRole(String metalake, String role) { String[] listRoleNames(String metalake) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofRoleNamespace(metalake); return store.list(namespace, RoleEntity.class, Entity.EntityType.ROLE).stream() .map(Role::name) .toArray(String[]::new); - } catch (NoSuchEntityException e) { - LOG.warn("Metalake {} does not exist", metalake, e); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalake); } catch (IOException ioe) { LOG.error("Listing user under metalake {} failed due to storage issues", metalake, ioe); throw new RuntimeException(ioe); @@ -154,7 +151,7 @@ String[] listRoleNames(String metalake) { String[] listRoleNamesByObject(String metalake, MetadataObject object) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.relationOperations() .listEntitiesByRelation( diff --git a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java index cd852ab66a7..905b100a652 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.authorization; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.collect.Lists; import java.io.IOException; import java.time.Instant; @@ -27,6 +29,7 @@ import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -62,7 +65,7 @@ class UserGroupManager { User addUser(String metalake, String name) throws UserAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); UserEntity userEntity = UserEntity.builder() .withId(idGenerator.nextId()) @@ -90,7 +93,7 @@ User addUser(String metalake, String name) throws UserAlreadyExistsException { boolean removeUser(String metalake, String user) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.delete(AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER); } catch (IOException ioe) { LOG.error( @@ -101,7 +104,7 @@ boolean removeUser(String metalake, String user) { User getUser(String metalake, String user) throws NoSuchUserException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.get( AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER, UserEntity.class); @@ -127,7 +130,7 @@ User[] listUsers(String metalake) { Group addGroup(String metalake, String group) throws GroupAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); GroupEntity groupEntity = GroupEntity.builder() .withId(idGenerator.nextId()) @@ -155,7 +158,7 @@ Group addGroup(String metalake, String group) throws GroupAlreadyExistsException boolean removeGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.delete(AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP); } catch (IOException ioe) { LOG.error( @@ -169,7 +172,7 @@ boolean removeGroup(String metalake, String group) { Group getGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.get( AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP, GroupEntity.class); @@ -194,7 +197,7 @@ String[] listGroupNames(String metalake) { private User[] listUsersInternal(String metalake, boolean allFields) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofUserNamespace(metalake); return store @@ -211,7 +214,7 @@ private User[] listUsersInternal(String metalake, boolean allFields) { private Group[] listGroupInternal(String metalake, boolean allFields) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofGroupNamespace(metalake); return store .list(namespace, GroupEntity.class, EntityType.GROUP, allFields) diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index 9113099e60f..dbb49c29732 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -22,6 +22,8 @@ import static org.apache.gravitino.StringIdentifier.DUMMY_ID; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; +import static org.apache.gravitino.metalake.MetalakeManager.metalakeInUse; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -105,7 +107,6 @@ public class CatalogManager implements CatalogDispatcher, Closeable { private static final String CATALOG_DOES_NOT_EXIST_MSG = "Catalog %s does not exist"; - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private static final PropertiesMetadata BASIC_CATALOG_PROPERTIES_METADATA = new BaseCatalogPropertiesMetadata() { @Override @@ -118,8 +119,8 @@ protected Map> specificPropertyEntries() { public static boolean catalogInUse(EntityStore store, NameIdentifier ident) throws NoSuchMetalakeException, NoSuchCatalogException { - // todo: check if the metalake is in use - return getInUseValue(store, ident); + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + return metalakeInUse(store, metalakeIdent) && getInUseValue(store, ident); } /** Wrapper class for a catalog instance and its class loader. */ @@ -292,7 +293,7 @@ public void close() { @Override public NameIdentifier[] listCatalogs(Namespace namespace) throws NoSuchMetalakeException { NameIdentifier metalakeIdent = NameIdentifier.of(namespace.levels()); - checkMetalakeExists(metalakeIdent); + checkMetalake(NameIdentifier.of(namespace.level(0)), store); try { return store.list(namespace, CatalogEntity.class, EntityType.CATALOG).stream() @@ -308,7 +309,7 @@ public NameIdentifier[] listCatalogs(Namespace namespace) throws NoSuchMetalakeE @Override public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeException { NameIdentifier metalakeIdent = NameIdentifier.of(namespace.levels()); - checkMetalakeExists(metalakeIdent); + checkMetalake(metalakeIdent, store); try { List catalogEntities = @@ -333,6 +334,9 @@ public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeExce */ @Override public Catalog loadCatalog(NameIdentifier ident) throws NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + return loadCatalogAndWrap(ident).catalog; } @@ -356,6 +360,9 @@ public Catalog createCatalog( String comment, Map properties) throws NoSuchMetalakeException, CatalogAlreadyExistsException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + Map mergedConfig = buildCatalogConf(provider, properties); long uid = idGenerator.nextId(); @@ -382,12 +389,6 @@ public Catalog createCatalog( boolean needClean = true; try { - NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); - if (!store.exists(metalakeIdent, EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } - store.put(e, false /* overwrite */); CatalogWrapper wrapper = catalogCache.get(ident, id -> createCatalogWrapper(e, mergedConfig)); @@ -441,11 +442,9 @@ public void testConnection( String comment, Map properties) { NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); - try { - if (!store.exists(metalakeIdent, EntityType.METALAKE)) { - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } + checkMetalake(metalakeIdent, store); + try { if (store.exists(ident, EntityType.CATALOG)) { throw new CatalogAlreadyExistsException("Catalog %s already exists", ident); } @@ -491,6 +490,9 @@ public void testConnection( @Override public void activateCatalog(NameIdentifier ident) throws NoSuchCatalogException, NonInUseEntityException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + try { boolean inUse = catalogInUse(store, ident); if (!inUse) { @@ -519,6 +521,9 @@ public void activateCatalog(NameIdentifier ident) @Override public void deactivateCatalog(NameIdentifier ident) throws NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + try { boolean inUse = catalogInUse(store, ident); if (inUse) { @@ -745,17 +750,6 @@ private Pair, Map> getCatalogAlterProperty( return Pair.of(upserts, deletes); } - private void checkMetalakeExists(NameIdentifier ident) throws NoSuchMetalakeException { - try { - if (!store.exists(ident, EntityType.METALAKE)) { - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, ident); - } - } catch (IOException e) { - LOG.error("Failed to do storage operation", e); - throw new RuntimeException(e); - } - } - private CatalogWrapper loadCatalogInternal(NameIdentifier ident) throws NoSuchCatalogException { try { CatalogEntity entity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); diff --git a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java index 830a94b4f60..d4778b2ff90 100644 --- a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java +++ b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java @@ -89,7 +89,7 @@ default Object getOrDefault(Map properties, String propertyName) throw new IllegalArgumentException("Property is not defined: " + propertyName); } - if (properties.containsKey(propertyName)) { + if (properties != null && properties.containsKey(propertyName)) { return propertyEntries().get(propertyName).decode(properties.get(propertyName)); } return propertyEntries().get(propertyName).getDefaultValue(); diff --git a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java index 3c242bd56fe..2132d71edca 100644 --- a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java @@ -27,8 +27,10 @@ import org.apache.gravitino.authorization.AccessControlDispatcher; import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.OwnerManager; +import org.apache.gravitino.exceptions.EntityInUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.metalake.MetalakeDispatcher; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.PrincipalUtils; @@ -87,8 +89,19 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) } @Override - public boolean dropMetalake(NameIdentifier ident) { - return dispatcher.dropMetalake(ident); + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, EntityInUseException { + return dispatcher.dropMetalake(ident, force); + } + + @Override + public void activateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.activateMetalake(ident); + } + + @Override + public void deactivateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.deactivateMetalake(ident); } @Override diff --git a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java index 33005893f4c..e39489e0417 100644 --- a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java @@ -23,8 +23,10 @@ import org.apache.gravitino.Metalake; import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.exceptions.EntityInUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.listener.api.event.AlterMetalakeEvent; import org.apache.gravitino.listener.api.event.AlterMetalakeFailureEvent; import org.apache.gravitino.listener.api.event.CreateMetalakeEvent; @@ -129,9 +131,10 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) } @Override - public boolean dropMetalake(NameIdentifier ident) { + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, EntityInUseException { try { - boolean isExists = dispatcher.dropMetalake(ident); + boolean isExists = dispatcher.dropMetalake(ident, force); eventBus.dispatchEvent( new DropMetalakeEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); return isExists; @@ -141,4 +144,16 @@ public boolean dropMetalake(NameIdentifier ident) { throw e; } } + + @Override + public void activateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + // todo: support activate metalake event + dispatcher.activateMetalake(ident); + } + + @Override + public void deactivateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + // todo: support deactivate metalake event + dispatcher.deactivateMetalake(ident); + } } diff --git a/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java b/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java index bab042d378f..4e2ca045f2e 100644 --- a/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java +++ b/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java @@ -25,13 +25,13 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.apache.gravitino.Audit; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; import org.apache.gravitino.Field; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.Metalake; -import org.apache.gravitino.StringIdentifier; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.metalake.MetalakePropertiesMetadata; /** Base implementation of a Metalake entity. */ @EqualsAndHashCode @@ -52,6 +52,8 @@ public class BaseMetalake implements Metalake, Entity, Auditable, HasIdentifier public static final Field SCHEMA_VERSION = Field.required("version", SchemaVersion.class, "The version of the schema for the metalake"); + public static final PropertiesMetadata PROPERTIES_METADATA = new MetalakePropertiesMetadata(); + private Long id; private String name; @@ -87,10 +89,10 @@ public Map fields() { /** * The audit information of the metalake. * - * @return The audit information as an {@link Audit} instance. + * @return The audit information as an {@link AuditInfo} instance. */ @Override - public Audit auditInfo() { + public AuditInfo auditInfo() { return auditInfo; } @@ -141,7 +143,11 @@ public EntityType type() { */ @Override public Map properties() { - return StringIdentifier.newPropertiesWithoutId(properties); + return properties; + } + + public PropertiesMetadata propertiesMetadata() { + return PROPERTIES_METADATA; } /** Builder class for creating instances of {@link BaseMetalake}. */ diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java index 54298c0efdd..a224bcd003a 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java @@ -18,9 +18,12 @@ */ package org.apache.gravitino.metalake; +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; + import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.Map; import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; @@ -28,13 +31,16 @@ import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; -import org.apache.gravitino.StringIdentifier; import org.apache.gravitino.exceptions.AlreadyExistsException; +import org.apache.gravitino.exceptions.EntityInUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; +import org.apache.gravitino.exceptions.NonInUseEntityException; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.BaseMetalake; +import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.meta.SchemaVersion; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.utils.PrincipalUtils; @@ -63,6 +69,48 @@ public MetalakeManager(EntityStore store, IdGenerator idGenerator) { this.idGenerator = idGenerator; } + /** + * Check whether the metalake is available + * + * @param ident The identifier of the Metalake to check. + * @param store The EntityStore to use for managing Metalakes. + * @throws NoSuchMetalakeException If the Metalake with the given identifier does not exist. + * @throws NonInUseEntityException If the Metalake is not in use. + */ + public static void checkMetalake(NameIdentifier ident, EntityStore store) + throws NoSuchMetalakeException, NonInUseEntityException { + boolean metalakeInUse = metalakeInUse(store, ident); + if (!metalakeInUse) { + throw new NonInUseEntityException( + "Metalake %s is not in use, please activate it first", ident); + } + } + + /** + * Return true if the metalake is in used, false otherwise. + * + * @param store The EntityStore to use for managing Metalakes. + * @param ident The identifier of the Metalake to check. + * @return True if the metalake is in use, false otherwise. + * @throws NoSuchMetalakeException If the Metalake with the given identifier does not exist. + */ + public static boolean metalakeInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException { + try { + BaseMetalake metalake = store.get(ident, EntityType.METALAKE, BaseMetalake.class); + return (boolean) + metalake.propertiesMetadata().getOrDefault(metalake.properties(), PROPERTY_IN_USE); + + } catch (NoSuchEntityException e) { + LOG.warn("Metalake {} does not exist", ident, e); + throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, ident); + + } catch (IOException e) { + LOG.error("Failed to do store operation", e); + throw new RuntimeException(e); + } + } + /** * Lists all available Metalakes. * @@ -72,8 +120,9 @@ public MetalakeManager(EntityStore store, IdGenerator idGenerator) { @Override public BaseMetalake[] listMetalakes() { try { - return store.list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE).stream() - .toArray(BaseMetalake[]::new); + return store + .list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE) + .toArray(new BaseMetalake[0]); } catch (IOException ioe) { LOG.error("Listing Metalakes failed due to storage issues.", ioe); throw new RuntimeException(ioe); @@ -116,14 +165,13 @@ public BaseMetalake createMetalake( NameIdentifier ident, String comment, Map properties) throws MetalakeAlreadyExistsException { long uid = idGenerator.nextId(); - StringIdentifier stringId = StringIdentifier.fromId(uid); BaseMetalake metalake = BaseMetalake.builder() .withId(uid) .withName(ident.name()) .withComment(comment) - .withProperties(StringIdentifier.newPropertiesWithId(stringId, properties)) + .withProperties(properties) .withVersion(SchemaVersion.V_0_1) .withAuditInfo( AuditInfo.builder() @@ -158,28 +206,17 @@ public BaseMetalake createMetalake( public BaseMetalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException { try { + if (!metalakeInUse(store, ident)) { + throw new EntityInUseException( + "Metalake %s is not in use, please activate it first", ident); + } + return store.update( ident, BaseMetalake.class, EntityType.METALAKE, metalake -> { - BaseMetalake.Builder builder = - BaseMetalake.builder() - .withId(metalake.id()) - .withName(metalake.name()) - .withComment(metalake.comment()) - .withProperties(metalake.properties()) - .withVersion(metalake.getVersion()); - - AuditInfo newInfo = - AuditInfo.builder() - .withCreator(metalake.auditInfo().creator()) - .withCreateTime(metalake.auditInfo().createTime()) - .withLastModifier( - metalake.auditInfo().creator()) /*TODO: Use real user later on. */ - .withLastModifiedTime(Instant.now()) - .build(); - builder.withAuditInfo(newInfo); + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); Map newProps = metalake.properties() == null @@ -204,23 +241,112 @@ public BaseMetalake alterMetalake(NameIdentifier ident, MetalakeChange... change } } - /** - * Deletes a Metalake. - * - * @param ident The identifier of the Metalake to be deleted. - * @return `true` if the Metalake was successfully deleted, `false` otherwise. - * @throws RuntimeException If deleting the Metalake encounters storage issues. - */ @Override - public boolean dropMetalake(NameIdentifier ident) { + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, EntityInUseException { try { - return store.delete(ident, EntityType.METALAKE); - } catch (IOException ioe) { - LOG.error("Deleting metalake {} failed due to storage issues", ident, ioe); - throw new RuntimeException(ioe); + boolean inUse = metalakeInUse(store, ident); + if (inUse && !force) { + throw new EntityInUseException( + "Metalake %s is in use, please deactivate it first or use force option", ident); + } + + List catalogEntities = + store.list(Namespace.of(ident.name()), CatalogEntity.class, EntityType.CATALOG); + if (!catalogEntities.isEmpty() && !force) { + throw new NonEmptyEntityException( + "Metalake %s has catalogs, please drop them first or use force option", ident); + } + + // If reached here, it implies that the metalake is not in use or force is true. + if (inUse) { + // force is true, so deactivate the metalake first. + deactivateMetalake(ident); + } + + return store.delete(ident, EntityType.METALAKE, true); + } catch (NoSuchMetalakeException e) { + return false; + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void activateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + try { + + boolean inUse = metalakeInUse(store, ident); + if (!inUse) { + store.update( + ident, + BaseMetalake.class, + EntityType.METALAKE, + metalake -> { + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); + + Map newProps = + metalake.properties() == null + ? Maps.newHashMap() + : Maps.newHashMap(metalake.properties()); + newProps.put(PROPERTY_IN_USE, "true"); + builder.withProperties(newProps); + + return builder.build(); + }); + } + } catch (IOException e) { + throw new RuntimeException(e); } } + @Override + public void deactivateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + try { + boolean inUse = metalakeInUse(store, ident); + if (inUse) { + store.update( + ident, + BaseMetalake.class, + EntityType.METALAKE, + metalake -> { + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); + + Map newProps = + metalake.properties() == null + ? Maps.newHashMap() + : Maps.newHashMap(metalake.properties()); + newProps.put(PROPERTY_IN_USE, "false"); + builder.withProperties(newProps); + + return builder.build(); + }); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private BaseMetalake.Builder newMetalakeBuilder(BaseMetalake metalake) { + BaseMetalake.Builder builder = + BaseMetalake.builder() + .withId(metalake.id()) + .withName(metalake.name()) + .withComment(metalake.comment()) + .withProperties(metalake.properties()) + .withVersion(metalake.getVersion()); + + AuditInfo newInfo = + AuditInfo.builder() + .withCreator(metalake.auditInfo().creator()) + .withCreateTime(metalake.auditInfo().createTime()) + .withLastModifier(metalake.auditInfo().creator()) /*TODO: Use real user later on. */ + .withLastModifiedTime(Instant.now()) + .build(); + return builder.withAuditInfo(newInfo); + } + /** * Updates an entity with the provided changes. * diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java index b0c8cbf1766..a2c716ae9a2 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java @@ -19,16 +19,25 @@ package org.apache.gravitino.metalake; import static org.apache.gravitino.Entity.SYSTEM_METALAKE_RESERVED_NAME; +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import static org.apache.gravitino.meta.BaseMetalake.PROPERTIES_METADATA; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import java.util.Arrays; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Metalake; import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.exceptions.EntityInUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; +import org.apache.gravitino.meta.BaseMetalake; public class MetalakeNormalizeDispatcher implements MetalakeDispatcher { private static final Set RESERVED_WORDS = ImmutableSet.of(SYSTEM_METALAKE_RESERVED_NAME); @@ -57,7 +66,7 @@ public Metalake[] listMetalakes() { @Override public Metalake loadMetalake(NameIdentifier ident) throws NoSuchMetalakeException { - return dispatcher.loadMetalake(ident); + return newMetalakeWithResolvedProperties((BaseMetalake) dispatcher.loadMetalake(ident)); } @Override @@ -70,6 +79,7 @@ public Metalake createMetalake( NameIdentifier ident, String comment, Map properties) throws MetalakeAlreadyExistsException { validateMetalakeName(ident.name()); + validatePropertyForCreate(PROPERTIES_METADATA, properties); return dispatcher.createMetalake(ident, comment, properties); } @@ -83,6 +93,10 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) validateMetalakeName(((MetalakeChange.RenameMetalake) change).getNewName()); } }); + Pair, Map> alterProperty = + getMetalakeAlterProperty(changes); + validatePropertyForAlter( + PROPERTIES_METADATA, alterProperty.getLeft(), alterProperty.getRight()); return dispatcher.alterMetalake(ident, changes); } @@ -93,6 +107,22 @@ public boolean dropMetalake(NameIdentifier ident) { return dispatcher.dropMetalake(ident); } + @Override + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, EntityInUseException { + return dispatcher.dropMetalake(ident, force); + } + + @Override + public void activateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.activateMetalake(ident); + } + + @Override + public void deactivateMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.deactivateMetalake(ident); + } + private void validateMetalakeName(String name) { if (RESERVED_WORDS.contains(name)) { throw new IllegalArgumentException("The metalake name '" + name + "' is reserved."); @@ -101,4 +131,45 @@ private void validateMetalakeName(String name) { throw new IllegalArgumentException("The metalake name '" + name + "' is illegal."); } } + + private BaseMetalake newMetalakeWithResolvedProperties(BaseMetalake metalakeEntity) { + Map newProps = Maps.newHashMap(metalakeEntity.properties()); + newProps + .entrySet() + .removeIf(e -> metalakeEntity.propertiesMetadata().isHiddenProperty(e.getKey())); + newProps.putIfAbsent( + PROPERTY_IN_USE, + metalakeEntity.propertiesMetadata().getDefaultValue(PROPERTY_IN_USE).toString()); + + return BaseMetalake.builder() + .withId(metalakeEntity.id()) + .withName(metalakeEntity.name()) + .withComment(metalakeEntity.comment()) + .withProperties(newProps) + .withVersion(metalakeEntity.getVersion()) + .withAuditInfo(metalakeEntity.auditInfo()) + .build(); + } + + private Pair, Map> getMetalakeAlterProperty( + MetalakeChange... metalakeChanges) { + Map upserts = Maps.newHashMap(); + Map deletes = Maps.newHashMap(); + + Arrays.stream(metalakeChanges) + .forEach( + metalakeChange -> { + if (metalakeChange instanceof MetalakeChange.SetProperty) { + MetalakeChange.SetProperty setProperty = + (MetalakeChange.SetProperty) metalakeChange; + upserts.put(setProperty.getProperty(), setProperty.getValue()); + } else if (metalakeChange instanceof MetalakeChange.RemoveProperty) { + MetalakeChange.RemoveProperty removeProperty = + (MetalakeChange.RemoveProperty) metalakeChange; + deletes.put(removeProperty.getProperty(), removeProperty.getProperty()); + } + }); + + return Pair.of(upserts, deletes); + } } diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java new file mode 100644 index 00000000000..7a39b0b63f8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.metalake; + +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; + +public class MetalakePropertiesMetadata implements PropertiesMetadata { + + private static final Map> PROPERTY_ENTRIES; + + static { + List> propertyEntries = + ImmutableList.of( + PropertyEntry.booleanReservedPropertyEntry( + PROPERTY_IN_USE, + "The property indicating the catalog is in use", + true /* default value */, + false /* hidden */)); + + PROPERTY_ENTRIES = Maps.uniqueIndex(propertyEntries, PropertyEntry::getName); + } + + @Override + public Map> propertyEntries() { + return PROPERTY_ENTRIES; + } +} diff --git a/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java b/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java index bd00c7ff5e8..6c29491d878 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java +++ b/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java @@ -23,8 +23,10 @@ import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.annotation.Evolving; +import org.apache.gravitino.exceptions.EntityInUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Interface for supporting metalakes. It includes methods for listing, loading, creating, altering @@ -89,10 +91,69 @@ Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException; /** - * Drop a metalake with specified identifier. + * Drop a metalake with specified identifier. Please make sure: + * + *
    + *
  • There is no catalog in the metalake. Otherwise, a {@link NonEmptyEntityException} will be + * thrown. + *
  • The method {@link #deactivateMetalake(NameIdentifier)} has been called before dropping + * the metalake. Otherwise, a {@link EntityInUseException} will be thrown. + *
+ * + * It is equivalent to calling {@code dropMetalake(ident, false)}. + * + * @param ident The identifier of the metalake. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty. + * @throws EntityInUseException If the metalake is in use. + */ + default boolean dropMetalake(NameIdentifier ident) + throws NonEmptyEntityException, EntityInUseException { + return dropMetalake(ident, false); + } + + /** + * Drop a metalake with specified identifier. If the force flag is true, it will: + * + *
    + *
  • Cascade drop all sub-entities (tags, catalogs, schemas, tables, etc.) of the metalake in + * Gravitino store. + *
  • Drop the metalake even if it is in use. + *
  • External resources (e.g. database, table, etc.) associated with sub-entities will not be + * deleted unless it is managed (such as managed fileset). + *
* * @param ident The identifier of the metalake. + * @param force Whether to force the drop. * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws EntityInUseException If the metalake is in use and force is false. + */ + boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, EntityInUseException; + + /** + * Activate a metalake. If the metalake is already active, this method does nothing. + * + * @param ident The identifier of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. + */ + void activateMetalake(NameIdentifier ident) throws NoSuchMetalakeException; + + /** + * Deactivate a metalake. If the metalake is already inactive, this method does nothing. Once a + * metalake is deactivated: + * + *
    + *
  • It can only be listed, loaded, dropped, or activated. + *
  • Any other operations on the metalake will throw an {@link + * org.apache.gravitino.exceptions.NonInUseEntityException}. + *
  • Any operation on the sub-entities (catalogs, schemas, tables, etc.) will throw an {@link + * org.apache.gravitino.exceptions.NonInUseEntityException}. + *
+ * + * @param ident The identifier of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. */ - boolean dropMetalake(NameIdentifier ident); + void deactivateMetalake(NameIdentifier ident) throws NoSuchMetalakeException; } diff --git a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java b/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java index d8536fffbdb..e18b280e691 100644 --- a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java +++ b/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java @@ -20,7 +20,6 @@ package org.apache.gravitino.proto; import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.AuditInfo; /** A class for serializing and deserializing BaseMetalake objects. */ class BaseMetalakeSerDe implements ProtoSerDe { @@ -38,7 +37,7 @@ public Metalake serialize(org.apache.gravitino.meta.BaseMetalake baseMetalake) { Metalake.newBuilder() .setId(baseMetalake.id()) .setName(baseMetalake.name()) - .setAuditInfo(new AuditInfoSerDe().serialize((AuditInfo) baseMetalake.auditInfo())); + .setAuditInfo(new AuditInfoSerDe().serialize(baseMetalake.auditInfo())); if (baseMetalake.comment() != null) { builder.setComment(baseMetalake.comment()); diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index aaffd35b50a..1b1626de00e 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.tag; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -35,7 +37,6 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchTagException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; @@ -93,7 +94,7 @@ public Tag[] listTagsInfo(String metalake) { NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore @@ -114,7 +115,7 @@ public Tag createTag(String metalake, String name, String comment, Map { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); TagEntity tagEntity = TagEntity.builder() @@ -148,7 +149,7 @@ public Tag getTag(String metalake, String name) throws NoSuchTagException { ofTagIdent(metalake, name), LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.get( @@ -169,7 +170,7 @@ public Tag alterTag(String metalake, String name, TagChange... changes) NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.WRITE, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.update( @@ -195,7 +196,7 @@ public boolean deleteTag(String metalake, String name) { NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.WRITE, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.delete(ofTagIdent(metalake, name), Entity.EntityType.TAG); @@ -213,7 +214,7 @@ public MetadataObject[] listMetadataObjectsForTag(String metalake, String name) tagId, LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { if (!entityStore.exists(tagId, Entity.EntityType.TAG)) { @@ -356,19 +357,6 @@ public String[] associateTagsForMetadataObject( })); } - private static void checkMetalakeExists(String metalake, EntityStore entityStore) { - try { - NameIdentifier metalakeIdent = NameIdentifier.of(metalake); - if (!entityStore.exists(metalakeIdent, Entity.EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException("Metalake %s does not exist", metalakeIdent); - } - } catch (IOException ioe) { - LOG.error("Failed to check if metalake exists", ioe); - throw new RuntimeException(ioe); - } - } - public static Namespace ofTagNamespace(String metalake) { return Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.TAG_SCHEMA_NAME); } diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java index 319ac641f61..2b5377f7598 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java @@ -19,6 +19,7 @@ package org.apache.gravitino.listener.api.event; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -205,7 +206,7 @@ private MetalakeDispatcher mockMetalakeDispatcher() { when(dispatcher.createMetalake(any(NameIdentifier.class), any(String.class), any(Map.class))) .thenReturn(metalake); when(dispatcher.loadMetalake(any(NameIdentifier.class))).thenReturn(metalake); - when(dispatcher.dropMetalake(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.dropMetalake(any(NameIdentifier.class), anyBoolean())).thenReturn(true); when(dispatcher.listMetalakes()).thenReturn(null); when(dispatcher.alterMetalake(any(NameIdentifier.class), any(MetalakeChange.class))) .thenReturn(metalake); diff --git a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java index 96bc1ebdd82..9375852b9bb 100644 --- a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java +++ b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java @@ -173,6 +173,7 @@ public void testDropMetalake() { Assertions.assertEquals("comment", metalake.comment()); testProperties(props, metalake.properties()); + metalakeManager.deactivateMetalake(ident); boolean dropped = metalakeManager.dropMetalake(ident); Assertions.assertTrue(dropped, "metalake should be dropped"); diff --git a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java index 6502b6931ea..666ba9aaead 100644 --- a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java @@ -695,9 +695,7 @@ void testEntityDelete(String type) throws IOException { // metalake BaseMetalake metalakeNew = createBaseMakeLake( - RandomIdGenerator.INSTANCE.nextId(), - metalake.name(), - (AuditInfo) metalake.auditInfo()); + RandomIdGenerator.INSTANCE.nextId(), metalake.name(), metalake.auditInfo()); store.put(metalakeNew); // catalog CatalogEntity catalogNew = @@ -976,7 +974,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake1"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake1New.id(), "metalake2", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake1New.id(), "metalake2", e.auditInfo())); // Rename metalake3 --> metalake1 BaseMetalake metalake3New1 = @@ -986,7 +984,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake3"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake3New1.id(), "metalake1", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake3New1.id(), "metalake1", e.auditInfo())); // Rename metalake3 --> metalake2 BaseMetalake metalake3New2 = @@ -998,7 +996,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake3"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake3New2.id(), "metalake2", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake3New2.id(), "metalake2", e.auditInfo())); // Finally, only metalake2 and metalake1 are left. Assertions.assertDoesNotThrow( diff --git a/gradle.properties b/gradle.properties index 878f17d4603..18d12de7338 100644 --- a/gradle.properties +++ b/gradle.properties @@ -40,4 +40,4 @@ defaultScalaVersion = 2.12 pythonVersion = 3.8 # skipDockerTests is used to skip the tests that require Docker to be running. -skipDockerTests = false +skipDockerTests = true diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index 1e59a541841..ff0c685faab 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -343,6 +343,12 @@ public Response handle(OperationType op, String metalake, String parent, Excepti } else if (e instanceof NoSuchMetalakeException) { return Utils.notFound(errorMsg, e); + } else if (e instanceof NonInUseEntityException) { + return Utils.nonInUse(errorMsg, e); + + } else if (e instanceof EntityInUseException) { + return Utils.inUse(errorMsg, e); + } else { return super.handle(op, metalake, parent, e); } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java index fff86cf2e87..5f0bbefc9e1 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java @@ -25,12 +25,14 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -42,6 +44,7 @@ import org.apache.gravitino.dto.requests.MetalakeCreateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdatesRequest; +import org.apache.gravitino.dto.responses.BaseResponse; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.MetalakeListResponse; import org.apache.gravitino.dto.responses.MetalakeResponse; @@ -149,6 +152,64 @@ public Response loadMetalake(@PathParam("name") String metalakeName) { } } + @GET + @Path("{name}/activate") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "activate-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "activate-metalake", absolute = true) + public Response activateMetalake(@PathParam("name") String metalakeName) { + LOG.info("Received activate request for metalake: {}", metalakeName); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); + TreeLockUtils.doWithTreeLock( + identifier, + LockType.READ, + () -> { + metalakeDispatcher.activateMetalake(identifier); + return null; + }); + Response response = Utils.ok(new BaseResponse()); + LOG.info("Metalake activated: {}", metalakeName); + return response; + }); + + } catch (Exception e) { + return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); + } + } + + @GET + @Path("{name}/deactivate") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "deactivate-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "deactivate-metalake", absolute = true) + public Response deactivateMetalake(@PathParam("name") String metalakeName) { + LOG.info("Received deactivate request for metalake: {}", metalakeName); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); + TreeLockUtils.doWithTreeLock( + identifier, + LockType.READ, + () -> { + metalakeDispatcher.deactivateMetalake(identifier); + return null; + }); + Response response = Utils.ok(new BaseResponse()); + LOG.info("Metalake deactivated: {}", metalakeName); + return response; + }); + + } catch (Exception e) { + return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); + } + } + @PUT @Path("{name}") @Produces("application/vnd.gravitino.v1+json") @@ -186,7 +247,9 @@ public Response alterMetalake( @Produces("application/vnd.gravitino.v1+json") @Timed(name = "drop-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "drop-metalake", absolute = true) - public Response dropMetalake(@PathParam("name") String metalakeName) { + public Response dropMetalake( + @PathParam("name") String metalakeName, + @DefaultValue("false") @QueryParam("force") boolean force) { LOG.info("Received drop metalake request for metalake: {}", metalakeName); try { return Utils.doAs( @@ -195,7 +258,7 @@ public Response dropMetalake(@PathParam("name") String metalakeName) { NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); boolean dropped = TreeLockUtils.doWithRootTreeLock( - LockType.WRITE, () -> metalakeDispatcher.dropMetalake(identifier)); + LockType.WRITE, () -> metalakeDispatcher.dropMetalake(identifier, force)); if (!dropped) { LOG.warn("Failed to drop metalake by name {}", metalakeName); } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java index 3b73cf035be..9b265e016d2 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java @@ -22,6 +22,7 @@ import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -381,7 +382,7 @@ public void testAlterMetalake() { @Test public void testDropMetalake() { - when(metalakeManager.dropMetalake(any())).thenReturn(true); + when(metalakeManager.dropMetalake(any(), anyBoolean())).thenReturn(true); Response resp = target("/metalakes/test") .request(MediaType.APPLICATION_JSON_TYPE) @@ -396,7 +397,7 @@ public void testDropMetalake() { Assertions.assertTrue(dropped); // Test when failed to drop metalake - when(metalakeManager.dropMetalake(any())).thenReturn(false); + when(metalakeManager.dropMetalake(any(), anyBoolean())).thenReturn(false); Response resp2 = target("/metalakes/test") .request(MediaType.APPLICATION_JSON_TYPE) @@ -408,7 +409,9 @@ public void testDropMetalake() { Assertions.assertFalse(dropResponse2.dropped()); // Test throw an exception when deleting tenant. - doThrow(new RuntimeException("Internal error")).when(metalakeManager).dropMetalake(any()); + doThrow(new RuntimeException("Internal error")) + .when(metalakeManager) + .dropMetalake(any(), anyBoolean()); Response resp1 = target("/metalakes/test") diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java index 0c374e21ef1..d3e0cd27fb5 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java @@ -156,7 +156,7 @@ private static void dropMetalake() { if (!exists) { return; } - gravitinoClient.dropMetalake(metalakeName); + gravitinoClient.dropMetalake(metalakeName, true); } private static void createCatalog(