From 9d106b107a0f3efb9d7bb329d1cc52d34208da0e Mon Sep 17 00:00:00 2001 From: mchades Date: Wed, 16 Oct 2024 15:40:08 +0800 Subject: [PATCH 1/3] re-define drop metalake --- .../java/org/apache/gravitino/Metalake.java | 3 + .../apache/gravitino/SupportsMetalakes.java | 74 ++++++- .../exceptions/MetalakeInUseException.java | 36 ++++ .../exceptions/MetalakeNotInUseException.java | 36 ++++ .../integration/test/RangerHiveE2EIT.java | 6 +- .../integration/test/HadoopCatalogIT.java | 7 +- .../test/HadoopUserAuthenticationIT.java | 2 +- .../test/HadoopUserImpersonationIT.java | 5 +- .../hive/integration/test/CatalogHiveIT.java | 11 +- .../integration/test/ProxyCatalogHiveIT.java | 5 +- .../integration/test/CatalogDorisIT.java | 7 +- .../integration/test/AuditCatalogMysqlIT.java | 7 +- .../integration/test/CatalogMysqlIT.java | 6 +- .../integration/test/CatalogPostgreSqlIT.java | 6 +- .../integration/test/CatalogKafkaIT.java | 6 +- .../test/CatalogIcebergBaseIT.java | 6 +- .../integration/test/CatalogPaimonBaseIT.java | 6 +- .../gravitino/client/ErrorHandlers.java | 56 +++++ .../client/GravitinoAdminClient.java | 62 +++++- .../client/integration/test/AuditIT.java | 1 + .../client/integration/test/CatalogIT.java | 2 +- .../client/integration/test/MetalakeIT.java | 5 +- .../client/integration/test/TagIT.java | 2 +- .../AccessControlNotAllowIT.java | 3 +- .../authorization/CheckCurrentUserIT.java | 2 +- .../test/authorization/OwnerIT.java | 10 +- .../client/gravitino_admin_client.py | 44 +++- .../dto/requests/metalake_set_request.py | 41 ++++ .../gravitino/exceptions/base.py | 8 + .../handlers/metalake_error_handler.py | 9 + .../integration/auth/test_auth_common.py | 4 +- .../tests/integration/test_catalog.py | 4 +- .../tests/integration/test_fileset_catalog.py | 4 +- .../tests/integration/test_metalake.py | 6 +- .../tests/integration/test_schema.py | 4 +- .../test/GravitinoVirtualFileSystemIT.java | 2 +- .../dto/requests/MetalakeSetRequest.java | 57 +++++ .../authorization/AuthorizationUtils.java | 22 -- .../gravitino/authorization/RoleManager.java | 17 +- .../authorization/UserGroupManager.java | 19 +- .../gravitino/catalog/CatalogManager.java | 66 +++--- .../catalog/OperationDispatcher.java | 11 +- .../connector/PropertiesMetadata.java | 2 +- .../hook/MetalakeHookDispatcher.java | 17 +- .../listener/MetalakeEventDispatcher.java | 19 +- .../apache/gravitino/meta/BaseMetalake.java | 16 +- .../gravitino/metalake/MetalakeManager.java | 194 +++++++++++++++--- .../metalake/MetalakeNormalizeDispatcher.java | 73 ++++++- .../metalake/MetalakePropertiesMetadata.java | 50 +++++ .../gravitino/metalake/SupportsMetalakes.java | 65 +++++- .../gravitino/proto/BaseMetalakeSerDe.java | 3 +- .../org/apache/gravitino/tag/TagManager.java | 28 +-- .../listener/api/event/TestMetalakeEvent.java | 3 +- .../metalake/TestMetalakeManager.java | 1 + .../gravitino/storage/TestEntityStorage.java | 10 +- .../server/web/rest/ExceptionHandlers.java | 50 ++++- .../server/web/rest/MetalakeOperations.java | 106 +++++++++- .../web/rest/TestMetalakeOperations.java | 9 +- .../integration/test/TrinoQueryITBase.java | 2 +- web/web/src/lib/api/metalakes/index.js | 2 +- 60 files changed, 1098 insertions(+), 242 deletions(-) create mode 100644 api/src/main/java/org/apache/gravitino/exceptions/MetalakeInUseException.java create mode 100644 api/src/main/java/org/apache/gravitino/exceptions/MetalakeNotInUseException.java create mode 100644 clients/client-python/gravitino/dto/requests/metalake_set_request.py create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/MetalakeSetRequest.java create mode 100644 core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java diff --git a/api/src/main/java/org/apache/gravitino/Metalake.java b/api/src/main/java/org/apache/gravitino/Metalake.java index fb6fdbee094..4f95a8a2365 100644 --- a/api/src/main/java/org/apache/gravitino/Metalake.java +++ b/api/src/main/java/org/apache/gravitino/Metalake.java @@ -29,6 +29,9 @@ @Evolving public interface Metalake extends Auditable { + /** The property indicating the metalake is in use. */ + String PROPERTY_IN_USE = "in-use"; + /** * The name of the metalake. * diff --git a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java index 47cce7d3a4a..0508f5239a6 100644 --- a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java +++ b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java @@ -21,7 +21,10 @@ import java.util.Map; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Client interface for supporting metalakes. It includes methods for listing, loading, creating, @@ -38,7 +41,7 @@ public interface SupportsMetalakes { Metalake[] listMetalakes(); /** - * Load a metalake by its identifier. + * Load a metalake by its name. * * @param name the name of the metalake. * @return The metalake. @@ -62,7 +65,7 @@ default boolean metalakeExists(String name) { } /** - * Create a metalake with specified identifier. + * Create a metalake with specified name, comment and properties. * * @param name The name of the metalake. * @param comment The comment of the metalake. @@ -74,7 +77,7 @@ Metalake createMetalake(String name, String comment, Map propert throws MetalakeAlreadyExistsException; /** - * Alter a metalake with specified identifier. + * Alter a metalake with specified metalake name and changes. * * @param name The name of the metalake. * @param changes The changes to apply. @@ -86,10 +89,69 @@ Metalake alterMetalake(String name, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException; /** - * Drop a metalake with specified identifier. + * Drop a metalake with specified name. Please make sure: * - * @param name The identifier of the metalake. + * + * + * It is equivalent to calling {@code dropMetalake(ident, false)}. + * + * @param name The name of the metalake. * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty. + * @throws MetalakeInUseException If the metalake is in use. + */ + default boolean dropMetalake(String name) throws NonEmptyEntityException, MetalakeInUseException { + return dropMetalake(name, false); + } + + /** + * Drop a metalake with specified name. If the force flag is true, it will: + * + * + * + * If the force flag is false, it is equivalent to calling {@link #dropMetalake(String)}. + * + * @param name The name of the metalake. + * @param force Whether to force the drop. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws MetalakeInUseException If the metalake is in use and force is false. + */ + boolean dropMetalake(String name, boolean force) + throws NonEmptyEntityException, MetalakeInUseException; + + /** + * Enable a metalake. If the metalake is already in use, this method does nothing. + * + * @param name The name of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. + */ + void enableMetalake(String name) throws NoSuchMetalakeException; + + /** + * Disable a metalake. If the metalake is already disabled, this method does nothing. Once a + * metalake is disable: + * + * + * + * @param name The name of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. */ - boolean dropMetalake(String name); + void disableMetalake(String name) throws NoSuchMetalakeException; } diff --git a/api/src/main/java/org/apache/gravitino/exceptions/MetalakeInUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeInUseException.java new file mode 100644 index 00000000000..5184116d907 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeInUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a metalake is in use and cannot be deleted. */ +public class MetalakeInUseException extends InUseException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public MetalakeInUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/MetalakeNotInUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeNotInUseException.java new file mode 100644 index 00000000000..4ad6cc33a6c --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeNotInUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when operating on a metalake that is not in use. */ +public class MetalakeNotInUseException extends NotInUseException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public MetalakeNotInUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index 0f1f4c05abd..99b6095d6bd 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -191,6 +191,7 @@ public void stop() throws IOException { })); Arrays.stream(metalake.listCatalogs()) .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); } if (sparkSession != null) { @@ -269,10 +270,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index ef8f37187e1..49bd29b2ee7 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -88,7 +88,7 @@ public void stop() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (fileSystem != null) { fileSystem.close(); } @@ -104,10 +104,9 @@ protected void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java index d0de2972742..d074709bd80 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java @@ -654,6 +654,6 @@ void testUserImpersonation() { catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, filesetName)); catalog.asSchemas().dropSchema(SCHEMA_NAME, true); gravitinoMetalake.dropCatalog(catalogName, true); - adminClient.dropMetalake(metalakeName); + adminClient.dropMetalake(metalakeName, true); } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java index 9515b45b5dd..808aafbaef5 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java @@ -258,10 +258,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index 31493d54ba2..903f3fe8a41 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -229,7 +229,7 @@ public void stop() throws IOException { })); Arrays.stream(metalake.listCatalogs()) .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); } if (hiveClientPool != null) { hiveClientPool.close(); @@ -264,10 +264,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } @@ -1429,8 +1428,8 @@ void testDropAndRename() { client.createMetalake(metalakeName1, "comment", Collections.emptyMap()); client.createMetalake(metalakeName2, "comment", Collections.emptyMap()); - client.dropMetalake(metalakeName1); - client.dropMetalake(metalakeName2); + client.dropMetalake(metalakeName1, true); + client.dropMetalake(metalakeName2, true); client.createMetalake(metalakeName1, "comment", Collections.emptyMap()); diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java index d328a44dc64..b7d61582efb 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java @@ -389,10 +389,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); + client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(METALAKE_NAME); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(METALAKE_NAME, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index b08aa491600..96b92b6969d 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -126,7 +126,7 @@ public void startup() throws IOException { public void stop() { clearTableAndSchema(); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); } @AfterEach @@ -143,10 +143,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetaLakes = client.listMetalakes(); assertEquals(0, gravitinoMetaLakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - assertEquals(createdMetalake, loadMetalake); + assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java index 784361d6407..b65d82c2103 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java @@ -77,7 +77,7 @@ public void startIntegrationTest() throws Exception { @AfterAll public void stopIntegrationTest() throws IOException, InterruptedException { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); mysqlService.close(); super.stopIntegrationTest(); } @@ -169,10 +169,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index 60af07011e8..f6b91b00ee4 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -144,6 +144,7 @@ public void stop() { clearTableAndSchema(); metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); mysqlService.close(); } @@ -167,10 +168,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index 0d1292c67a2..f22073c7815 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -130,6 +130,7 @@ public void stop() { } metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); postgreSqlService.close(); } @@ -153,10 +154,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index 907b00733bc..dc91a3dda57 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -126,6 +126,7 @@ public void shutdown() { metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); })); + client.disableMetalake(METALAKE_NAME); client.dropMetalake(METALAKE_NAME); if (adminClient != null) { adminClient.close(); @@ -554,10 +555,9 @@ private TopicDescription getTopicDesc(String topicName) } private void createMetalake() { - GravitinoMetalake createdMetalake = - client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); + client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(METALAKE_NAME); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(METALAKE_NAME, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index 9fcc93451ca..7c5d93362f6 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -140,6 +140,7 @@ public void stop() throws Exception { clearTableAndSchema(); metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); } finally { if (spark != null) { @@ -197,10 +198,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java index ed90745a785..bf20ad51336 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java @@ -138,6 +138,7 @@ public void stop() { clearTableAndSchema(); metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); if (spark != null) { spark.close(); @@ -878,10 +879,9 @@ private void clearTableAndSchema() { } private void createMetalake() { - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java index a2ff07e27ad..6d91767fdca 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java @@ -37,6 +37,8 @@ import org.apache.gravitino.exceptions.IllegalPrivilegeException; import org.apache.gravitino.exceptions.InUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchGroupException; @@ -279,6 +281,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -327,6 +334,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -375,6 +387,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -426,6 +443,9 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogInUseException.class.getSimpleName())) { throw new CatalogInUseException(errorMessage); + } else if (errorResponse.getType().equals(MetalakeInUseException.class.getSimpleName())) { + throw new MetalakeInUseException(errorMessage); + } else { throw new InUseException(errorMessage); } @@ -434,6 +454,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -466,6 +491,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); + case ErrorConstants.IN_USE_CODE: + throw new MetalakeInUseException(errorMessage); + default: super.accept(errorResponse); } @@ -560,6 +588,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -606,6 +639,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -645,6 +683,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -683,6 +724,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -732,6 +776,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.FORBIDDEN_CODE: throw new ForbiddenException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -780,6 +827,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -823,6 +873,9 @@ public void accept(ErrorResponse errorResponse) { throw new AlreadyExistsException(errorMessage); } + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -856,6 +909,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java index 936edcf3c2a..32c425fc76a 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -31,10 +32,13 @@ import org.apache.gravitino.dto.requests.MetalakeUpdateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdatesRequest; import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.dto.responses.MetalakeListResponse; import org.apache.gravitino.dto.responses.MetalakeResponse; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Apache Gravitino Client for the administrator to interact with the Gravitino API, allowing the @@ -145,17 +149,35 @@ public GravitinoMetalake alterMetalake(String name, MetalakeChange... changes) } /** - * Drops a specific Metalake using the Gravitino API. + * Drop a metalake with specified name. If the force flag is true, it will: * - * @param name The name of the Metalake to be dropped. - * @return True if the Metalake was successfully dropped, false if the Metalake does not exist. + * + * + * If the force flag is false, it is equivalent to calling {@link #dropMetalake(String)}. + * + * @param name The name of the metalake. + * @param force Whether to force the drop. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws MetalakeInUseException If the metalake is in use and force is false. */ @Override - public boolean dropMetalake(String name) { + public boolean dropMetalake(String name, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { checkMetalakeName(name); + Map params = new HashMap<>(); + params.put("force", String.valueOf(force)); + DropResponse resp = restClient.delete( API_METALAKES_IDENTIFIER_PATH + name, + params, DropResponse.class, Collections.emptyMap(), ErrorHandlers.metalakeErrorHandler()); @@ -163,6 +185,38 @@ public boolean dropMetalake(String name) { return resp.dropped(); } + @Override + public void enableMetalake(String name) throws NoSuchMetalakeException { + ErrorResponse resp = + restClient.get( + API_METALAKES_IDENTIFIER_PATH + name + "/activate", + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.metalakeErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.metalakeErrorHandler().accept(resp); + } + + @Override + public void disableMetalake(String name) throws NoSuchMetalakeException { + ErrorResponse resp = + restClient.get( + API_METALAKES_IDENTIFIER_PATH + name + "/deactivate", + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.metalakeErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.metalakeErrorHandler().accept(resp); + } + /** * Creates a new builder for constructing a GravitinoClient. * diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java index c438e0fca1e..fd1fac54f2f 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java @@ -60,6 +60,7 @@ public void testAuditMetalake() { metaLake = client.alterMetalake(metalakeAuditName, changes); Assertions.assertEquals(expectUser, metaLake.auditInfo().creator()); Assertions.assertEquals(expectUser, metaLake.auditInfo().lastModifier()); + Assertions.assertDoesNotThrow(() -> client.disableMetalake(newName)); Assertions.assertTrue(client.dropMetalake(newName), "metaLake should be dropped"); Assertions.assertFalse(client.dropMetalake(newName), "metalake should be non-existent"); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java index a29ef732094..5360f9f7816 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java @@ -81,7 +81,7 @@ public void startUp() { @AfterAll public void tearDown() { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java index fb9efd2ca7f..2922154f319 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.client.integration.test; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -182,6 +183,7 @@ public void testCreateMetalakeWithChinese() { public void testDropMetalakes() { GravitinoMetalake metalakeA = client.createMetalake(metalakeNameA, "metalake A comment", Collections.emptyMap()); + assertDoesNotThrow(() -> client.disableMetalake(metalakeA.name())); assertTrue(client.dropMetalake(metalakeA.name()), "metaLake should be dropped"); NameIdentifier id = NameIdentifier.of(metalakeNameA); assertThrows( @@ -205,12 +207,13 @@ public void testUpdateMetalakeWithNullableComment() { new MetalakeChange[] {MetalakeChange.updateComment("new metalake comment")}; GravitinoMetalake updatedMetalake = client.alterMetalake(metalakeNameA, changes); assertEquals("new metalake comment", updatedMetalake.comment()); - client.dropMetalake(metalakeNameA); + assertTrue(client.dropMetalake(metalakeNameA, true)); } public void dropMetalakes() { GravitinoMetalake[] metaLakes = client.listMetalakes(); for (GravitinoMetalake metalake : metaLakes) { + assertDoesNotThrow(() -> client.disableMetalake(metalake.name())); assertTrue(client.dropMetalake(metalake.name())); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java index f5b5e1dab56..dc82dfd67df 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java @@ -112,7 +112,7 @@ public void tearDown() { relationalCatalog.asTableCatalog().dropTable(NameIdentifier.of(schema.name(), table.name())); relationalCatalog.asSchemas().dropSchema(schema.name(), true); metalake.dropCatalog(relationalCatalog.name(), true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java index a6817b27418..1dcdd27a3e0 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.integration.test.authorization; +package org.apache.gravitino.client.integration.test.authorization; import com.google.common.collect.Lists; import java.util.Collections; @@ -148,6 +148,7 @@ public void testNotAllowFilter() { Assertions.assertTrue( e.getMessage().contains("You should set 'gravitino.authorization.enable'")); + client.disableMetalake(metalakeTestName); client.dropMetalake(metalakeTestName); } } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java index f5615beae06..a7339ba0db1 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java @@ -98,7 +98,7 @@ public void startIntegrationTest() throws Exception { @AfterAll public void tearDown() { if (client != null) { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); client.close(); client = null; } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index 99f1e830692..daac8002ab9 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -169,7 +169,7 @@ public void testCreateFileset() { catalog.asFilesetCatalog().dropFileset(fileIdent); catalog.asSchemas().dropSchema("schema_owner", true); metalake.dropCatalog(catalogNameA, true); - client.dropMetalake(metalakeNameA); + client.dropMetalake(metalakeNameA, true); } @Test @@ -220,7 +220,7 @@ public void testCreateTopic() { // Clean up catalogB.asTopicCatalog().dropTopic(topicIdent); metalake.dropCatalog(catalogNameB, true); - client.dropMetalake(metalakeNameB); + client.dropMetalake(metalakeNameB, true); } @Test @@ -255,7 +255,7 @@ public void testCreateRole() { // Clean up metalake.deleteRole("role_owner"); - client.dropMetalake(metalakeNameC); + client.dropMetalake(metalakeNameC, true); } @Test @@ -321,7 +321,7 @@ public void testCreateTable() { catalog.asTableCatalog().dropTable(tableIdent); catalog.asSchemas().dropSchema("schema_owner", true); metalake.dropCatalog(catalogNameD, true); - client.dropMetalake(metalakeNameD); + client.dropMetalake(metalakeNameD, true); } @Test @@ -352,6 +352,6 @@ public void testOwnerWithException() { () -> metalake.setOwner(metalakeObject, "not-existed", Owner.Type.USER)); // Cleanup - client.dropMetalake(metalakeNameE); + client.dropMetalake(metalakeNameE, true); } } diff --git a/clients/client-python/gravitino/client/gravitino_admin_client.py b/clients/client-python/gravitino/client/gravitino_admin_client.py index b730a765b21..85d9ff2f047 100644 --- a/clients/client-python/gravitino/client/gravitino_admin_client.py +++ b/clients/client-python/gravitino/client/gravitino_admin_client.py @@ -22,6 +22,7 @@ from gravitino.client.gravitino_metalake import GravitinoMetalake from gravitino.dto.dto_converters import DTOConverters from gravitino.dto.requests.metalake_create_request import MetalakeCreateRequest +from gravitino.dto.requests.metalake_set_request import MetalakeSetRequest from gravitino.dto.requests.metalake_updates_request import MetalakeUpdatesRequest from gravitino.dto.responses.drop_response import DropResponse from gravitino.dto.responses.metalake_list_response import MetalakeListResponse @@ -112,20 +113,59 @@ def alter_metalake(self, name: str, *changes: MetalakeChange) -> GravitinoMetala return GravitinoMetalake(metalake, self._rest_client) - def drop_metalake(self, name: str) -> bool: + def drop_metalake(self, name: str, force: bool = False) -> bool: """Drops a specific Metalake using the Gravitino API. Args: name: The name of the Metalake to be dropped. + force: Whether to force the drop operation. Returns: - True if the Metalake was successfully dropped, false otherwise. + True if the Metalake was successfully dropped, false if the Metalake does not exist. """ + params = {"force": str(force)} resp = self._rest_client.delete( self.API_METALAKES_IDENTIFIER_PATH + name, + params=params, error_handler=METALAKE_ERROR_HANDLER, ) drop_response = DropResponse.from_json(resp.body, infer_missing=True) return drop_response.dropped() + + def enable_metalake(self, name: str): + """Enable the metalake with specified name. If the metalake is already in use, this method does nothing. + + Args: + name: the name of the metalake. + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + """ + + metalake_enable_request = MetalakeSetRequest(in_use=True) + metalake_enable_request.validate() + + url = self.API_METALAKES_IDENTIFIER_PATH + name + self._rest_client.patch( + url, json=metalake_enable_request, error_handler=METALAKE_ERROR_HANDLER + ) + + def disable_metalake(self, name: str): + """Disable the metalake with specified name. If the metalake is already disabled, does nothing. + + Args: + name: the name of the metalake. + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + """ + + metalake_disable_request = MetalakeSetRequest(in_use=False) + metalake_disable_request.validate() + + url = self.API_METALAKES_IDENTIFIER_PATH + name + self._rest_client.patch( + url, json=metalake_disable_request, error_handler=METALAKE_ERROR_HANDLER + ) diff --git a/clients/client-python/gravitino/dto/requests/metalake_set_request.py b/clients/client-python/gravitino/dto/requests/metalake_set_request.py new file mode 100644 index 00000000000..a7663b0ed6c --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/metalake_set_request.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field + +from dataclasses_json import config + +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class MetalakeSetRequest(RESTRequest): + """Represents a request to set a metalake in use status.""" + + _in_use: bool = field(metadata=config(field_name="inUse")) + + def __init__(self, in_use: bool): + self._in_use = in_use + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if in_use is not set. + """ + if self._in_use is None: + raise ValueError('"in_use" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/exceptions/base.py b/clients/client-python/gravitino/exceptions/base.py index 2dff76ac4ad..cd71de2368c 100644 --- a/clients/client-python/gravitino/exceptions/base.py +++ b/clients/client-python/gravitino/exceptions/base.py @@ -97,6 +97,10 @@ class InUseException(GravitinoRuntimeException): """Base class for all exceptions thrown when an entity is in use and cannot be deleted.""" +class MetalakeInUseException(InUseException): + """An exception thrown when a metalake is in use and cannot be deleted.""" + + class CatalogInUseException(InUseException): """An Exception thrown when a catalog is in use and cannot be deleted.""" @@ -105,6 +109,10 @@ class NotInUseException(GravitinoRuntimeException): """Base class for all exceptions thrown when an entity is not in use.""" +class MetalakeNotInUseException(NotInUseException): + """An exception thrown when operating on not in use metalake.""" + + class CatalogNotInUseException(NotInUseException): """An exception thrown when operating on not in use catalog.""" diff --git a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py index 86dc575702e..60a7dd0ac36 100644 --- a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py +++ b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py @@ -21,6 +21,8 @@ from gravitino.exceptions.base import ( NoSuchMetalakeException, MetalakeAlreadyExistsException, + MetalakeInUseException, + MetalakeNotInUseException, ) @@ -33,9 +35,16 @@ def handle(self, error_response: ErrorResponse): if code == ErrorConstants.NOT_FOUND_CODE: raise NoSuchMetalakeException(error_message) + if code == ErrorConstants.ALREADY_EXISTS_CODE: raise MetalakeAlreadyExistsException(error_message) + if code == ErrorConstants.IN_USE_CODE: + raise MetalakeInUseException(error_message) + + if code == ErrorConstants.NOT_IN_USE_CODE: + raise MetalakeNotInUseException(error_message) + super().handle(error_response) diff --git a/clients/client-python/tests/integration/auth/test_auth_common.py b/clients/client-python/tests/integration/auth/test_auth_common.py index 2f1506319a7..ede3a4e2ad2 100644 --- a/clients/client-python/tests/integration/auth/test_auth_common.py +++ b/clients/client-python/tests/integration/auth/test_auth_common.py @@ -101,7 +101,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 58580b8c518..755b295b00f 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -92,7 +92,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 2696c170a0d..754735b16e8 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -128,7 +128,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/test_metalake.py b/clients/client-python/tests/integration/test_metalake.py index 794cb894d97..75d3a06f26c 100644 --- a/clients/client-python/tests/integration/test_metalake.py +++ b/clients/client-python/tests/integration/test_metalake.py @@ -118,7 +118,7 @@ def test_alter_metalake(self): self.assertTrue(self.metalake_properties_key1 not in metalake.properties()) def drop_metalake(self, metalake_name: str) -> bool: - return self.gravitino_admin_client.drop_metalake(metalake_name) + return self.gravitino_admin_client.drop_metalake(metalake_name, True) def test_drop_metalake(self): self.create_metalake(self.metalake_name) @@ -152,7 +152,9 @@ def test_load_metalakes(self): self.assertIsNotNone(metalake) self.assertEqual(metalake.name(), self.metalake_name) self.assertEqual(metalake.comment(), self.metalake_comment) - self.assertEqual(metalake.properties(), self.metalake_properties) + self.assertEqual( + metalake.properties(), {**self.metalake_properties, "in-use": "true"} + ) self.assertEqual(metalake.audit_info().creator(), "anonymous") def test_failed_load_metalakes(self): diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index 269693dcf25..c8a6b270b84 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -128,7 +128,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java index ef41637d6f9..064643b7931 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java @@ -92,7 +92,7 @@ public void tearDown() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/MetalakeSetRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/MetalakeSetRequest.java new file mode 100644 index 00000000000..be75b2d4a21 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/MetalakeSetRequest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to set a Metalake in use. */ +@Getter +@EqualsAndHashCode +@ToString +public class MetalakeSetRequest implements RESTRequest { + + private final boolean inUse; + + /** Default constructor for MetalakeSetRequest. */ + public MetalakeSetRequest() { + this(false); + } + + /** + * Constructor for MetalakeSetRequest. + * + * @param inUse The in use status to set. + */ + public MetalakeSetRequest(boolean inUse) { + this.inUse = inUse; + } + + /** + * Validates the request. No validation needed. + * + * @throws IllegalArgumentException If the request is invalid. + */ + @Override + public void validate() throws IllegalArgumentException { + // No validation needed + } +} diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index 81447abfd3b..1d5a2acf034 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -19,14 +19,12 @@ package org.apache.gravitino.authorization; import com.google.common.collect.Sets; -import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Set; import java.util.function.Consumer; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; -import org.apache.gravitino.EntityStore; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -40,12 +38,9 @@ import org.apache.gravitino.exceptions.IllegalPrivilegeException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchUserException; import org.apache.gravitino.utils.MetadataObjectUtil; import org.apache.gravitino.utils.NameIdentifierUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /* The utilization class of authorization module*/ public class AuthorizationUtils { @@ -53,8 +48,6 @@ public class AuthorizationUtils { static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in th metalake %s"; static final String GROUP_DOES_NOT_EXIST_MSG = "Group %s does not exist in th metalake %s"; static final String ROLE_DOES_NOT_EXIST_MSG = "Role %s does not exist in th metalake %s"; - private static final Logger LOG = LoggerFactory.getLogger(AuthorizationUtils.class); - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private static final Set FILESET_PRIVILEGES = Sets.immutableEnumSet( @@ -68,21 +61,6 @@ public class AuthorizationUtils { private AuthorizationUtils() {} - static void checkMetalakeExists(String metalake) throws NoSuchMetalakeException { - try { - EntityStore store = GravitinoEnv.getInstance().entityStore(); - - NameIdentifier metalakeIdent = NameIdentifier.of(metalake); - if (!store.exists(metalakeIdent, Entity.EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } - } catch (IOException e) { - LOG.error("Failed to do storage operation", e); - throw new RuntimeException(e); - } - } - public static void checkCurrentUser(String metalake, String user) { try { AccessControlDispatcher dispatcher = GravitinoEnv.getInstance().accessControlDispatcher(); diff --git a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java index dc675fdcef5..11c24102bca 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java @@ -19,6 +19,8 @@ package org.apache.gravitino.authorization; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.collect.Sets; import java.io.IOException; import java.time.Instant; @@ -33,7 +35,6 @@ import org.apache.gravitino.SupportsRelationOperations; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.meta.AuditInfo; @@ -52,7 +53,6 @@ class RoleManager { private static final Logger LOG = LoggerFactory.getLogger(RoleManager.class); - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private final EntityStore store; private final IdGenerator idGenerator; @@ -67,7 +67,7 @@ RoleEntity createRole( Map properties, List securableObjects) throws RoleAlreadyExistsException { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); RoleEntity roleEntity = RoleEntity.builder() .withId(idGenerator.nextId()) @@ -104,7 +104,7 @@ RoleEntity createRole( RoleEntity getRole(String metalake, String role) throws NoSuchRoleException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return getRoleEntity(AuthorizationUtils.ofRole(metalake, role)); } catch (NoSuchEntityException e) { LOG.warn("Role {} does not exist in the metalake {}", role, metalake, e); @@ -114,7 +114,7 @@ RoleEntity getRole(String metalake, String role) throws NoSuchRoleException { boolean deleteRole(String metalake, String role) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); NameIdentifier ident = AuthorizationUtils.ofRole(metalake, role); try { @@ -138,14 +138,11 @@ boolean deleteRole(String metalake, String role) { String[] listRoleNames(String metalake) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofRoleNamespace(metalake); return store.list(namespace, RoleEntity.class, Entity.EntityType.ROLE).stream() .map(Role::name) .toArray(String[]::new); - } catch (NoSuchEntityException e) { - LOG.warn("Metalake {} does not exist", metalake, e); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalake); } catch (IOException ioe) { LOG.error("Listing user under metalake {} failed due to storage issues", metalake, ioe); throw new RuntimeException(ioe); @@ -154,7 +151,7 @@ String[] listRoleNames(String metalake) { String[] listRoleNamesByObject(String metalake, MetadataObject object) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.relationOperations() .listEntitiesByRelation( diff --git a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java index cd852ab66a7..905b100a652 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.authorization; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.collect.Lists; import java.io.IOException; import java.time.Instant; @@ -27,6 +29,7 @@ import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -62,7 +65,7 @@ class UserGroupManager { User addUser(String metalake, String name) throws UserAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); UserEntity userEntity = UserEntity.builder() .withId(idGenerator.nextId()) @@ -90,7 +93,7 @@ User addUser(String metalake, String name) throws UserAlreadyExistsException { boolean removeUser(String metalake, String user) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.delete(AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER); } catch (IOException ioe) { LOG.error( @@ -101,7 +104,7 @@ boolean removeUser(String metalake, String user) { User getUser(String metalake, String user) throws NoSuchUserException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.get( AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER, UserEntity.class); @@ -127,7 +130,7 @@ User[] listUsers(String metalake) { Group addGroup(String metalake, String group) throws GroupAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); GroupEntity groupEntity = GroupEntity.builder() .withId(idGenerator.nextId()) @@ -155,7 +158,7 @@ Group addGroup(String metalake, String group) throws GroupAlreadyExistsException boolean removeGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.delete(AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP); } catch (IOException ioe) { LOG.error( @@ -169,7 +172,7 @@ boolean removeGroup(String metalake, String group) { Group getGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.get( AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP, GroupEntity.class); @@ -194,7 +197,7 @@ String[] listGroupNames(String metalake) { private User[] listUsersInternal(String metalake, boolean allFields) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofUserNamespace(metalake); return store @@ -211,7 +214,7 @@ private User[] listUsersInternal(String metalake, boolean allFields) { private Group[] listGroupInternal(String metalake, boolean allFields) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofGroupNamespace(metalake); return store .list(namespace, GroupEntity.class, EntityType.GROUP, allFields) diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index f03b500bb2d..aed3a676e4f 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -23,6 +23,8 @@ import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; import static org.apache.gravitino.connector.BaseCatalogPropertiesMetadata.BASIC_CATALOG_PROPERTIES_METADATA; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; +import static org.apache.gravitino.metalake.MetalakeManager.metalakeInUse; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -81,6 +83,7 @@ import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.GravitinoRuntimeException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; @@ -104,14 +107,18 @@ public class CatalogManager implements CatalogDispatcher, Closeable { private static final String CATALOG_DOES_NOT_EXIST_MSG = "Catalog %s does not exist"; - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private static final Logger LOG = LoggerFactory.getLogger(CatalogManager.class); - public static boolean catalogInUse(EntityStore store, NameIdentifier ident) - throws NoSuchMetalakeException, NoSuchCatalogException { - // todo: check if the metalake is in use - return getInUseValue(store, ident); + public static void checkCatalogInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException, NoSuchCatalogException, CatalogNotInUseException, + MetalakeNotInUseException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + + if (!getInUseValue(store, ident)) { + throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); + } } /** Wrapper class for a catalog instance and its class loader. */ @@ -284,7 +291,7 @@ public void close() { @Override public NameIdentifier[] listCatalogs(Namespace namespace) throws NoSuchMetalakeException { NameIdentifier metalakeIdent = NameIdentifier.of(namespace.levels()); - checkMetalakeExists(metalakeIdent); + checkMetalake(NameIdentifier.of(namespace.level(0)), store); try { return store.list(namespace, CatalogEntity.class, EntityType.CATALOG).stream() @@ -300,7 +307,7 @@ public NameIdentifier[] listCatalogs(Namespace namespace) throws NoSuchMetalakeE @Override public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeException { NameIdentifier metalakeIdent = NameIdentifier.of(namespace.levels()); - checkMetalakeExists(metalakeIdent); + checkMetalake(metalakeIdent, store); try { List catalogEntities = @@ -325,6 +332,9 @@ public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeExce */ @Override public Catalog loadCatalog(NameIdentifier ident) throws NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + return loadCatalogAndWrap(ident).catalog; } @@ -348,6 +358,9 @@ public Catalog createCatalog( String comment, Map properties) throws NoSuchMetalakeException, CatalogAlreadyExistsException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + Map mergedConfig = buildCatalogConf(provider, properties); long uid = idGenerator.nextId(); @@ -374,12 +387,6 @@ public Catalog createCatalog( boolean needClean = true; try { - NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); - if (!store.exists(metalakeIdent, EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } - store.put(e, false /* overwrite */); CatalogWrapper wrapper = catalogCache.get(ident, id -> createCatalogWrapper(e, mergedConfig)); @@ -433,11 +440,9 @@ public void testConnection( String comment, Map properties) { NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); - try { - if (!store.exists(metalakeIdent, EntityType.METALAKE)) { - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } + checkMetalake(metalakeIdent, store); + try { if (store.exists(ident, EntityType.CATALOG)) { throw new CatalogAlreadyExistsException("Catalog %s already exists", ident); } @@ -483,6 +488,9 @@ public void testConnection( @Override public void enableCatalog(NameIdentifier ident) throws NoSuchCatalogException, CatalogNotInUseException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + try { if (catalogInUse(store, ident)) { return; @@ -511,6 +519,9 @@ public void enableCatalog(NameIdentifier ident) @Override public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + try { if (!catalogInUse(store, ident)) { return; @@ -550,9 +561,7 @@ public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { @Override public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) throws NoSuchCatalogException, IllegalArgumentException { - if (!catalogInUse(store, ident)) { - throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); - } + checkCatalogInUse(store, ident); // There could be a race issue that someone is using the catalog from cache while we are // updating it. @@ -665,6 +674,12 @@ public CatalogWrapper loadCatalogAndWrap(NameIdentifier ident) throws NoSuchCata return catalogCache.get(ident, this::loadCatalogInternal); } + private static boolean catalogInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException, NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + return metalakeInUse(store, metalakeIdent) && getInUseValue(store, ident); + } + private static boolean getInUseValue(EntityStore store, NameIdentifier catalogIdent) { try { CatalogEntity catalogEntity = @@ -730,17 +745,6 @@ private Pair, Map> getCatalogAlterProperty( return Pair.of(upserts, deletes); } - private void checkMetalakeExists(NameIdentifier ident) throws NoSuchMetalakeException { - try { - if (!store.exists(ident, EntityType.METALAKE)) { - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, ident); - } - } catch (IOException e) { - LOG.error("Failed to do storage operation", e); - throw new RuntimeException(e); - } - } - private CatalogWrapper loadCatalogInternal(NameIdentifier ident) throws NoSuchCatalogException { try { CatalogEntity entity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); diff --git a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java index 88add624870..3e2ed6c1b18 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java @@ -18,7 +18,7 @@ */ package org.apache.gravitino.catalog; -import static org.apache.gravitino.catalog.CatalogManager.catalogInUse; +import static org.apache.gravitino.catalog.CatalogManager.checkCatalogInUse; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; import static org.apache.gravitino.utils.NameIdentifierUtil.getCatalogIdentifier; @@ -35,7 +35,6 @@ import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.PropertiesMetadata; import org.apache.gravitino.connector.capability.Capability; -import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.messaging.TopicChange; @@ -94,9 +93,7 @@ protected R doWithTable( protected R doWithCatalog( NameIdentifier ident, ThrowableFunction fn, Class ex) throws E { - if (!catalogInUse(store, ident)) { - throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); - } + checkCatalogInUse(store, ident); try { CatalogManager.CatalogWrapper c = catalogManager.loadCatalogAndWrap(ident); @@ -118,9 +115,7 @@ protected R doWithCatalog( Class ex1, Class ex2) throws E1, E2 { - if (!catalogInUse(store, ident)) { - throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); - } + checkCatalogInUse(store, ident); try { CatalogManager.CatalogWrapper c = catalogManager.loadCatalogAndWrap(ident); diff --git a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java index 830a94b4f60..d4778b2ff90 100644 --- a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java +++ b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java @@ -89,7 +89,7 @@ default Object getOrDefault(Map properties, String propertyName) throw new IllegalArgumentException("Property is not defined: " + propertyName); } - if (properties.containsKey(propertyName)) { + if (properties != null && properties.containsKey(propertyName)) { return propertyEntries().get(propertyName).decode(properties.get(propertyName)); } return propertyEntries().get(propertyName).getDefaultValue(); diff --git a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java index 3c242bd56fe..95554857a95 100644 --- a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java @@ -28,7 +28,9 @@ import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.OwnerManager; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.metalake.MetalakeDispatcher; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.PrincipalUtils; @@ -87,8 +89,19 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) } @Override - public boolean dropMetalake(NameIdentifier ident) { - return dispatcher.dropMetalake(ident); + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { + return dispatcher.dropMetalake(ident, force); + } + + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.enableMetalake(ident); + } + + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.disableMetalake(ident); } @Override diff --git a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java index 33005893f4c..23f9b328b47 100644 --- a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java @@ -24,7 +24,9 @@ import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.listener.api.event.AlterMetalakeEvent; import org.apache.gravitino.listener.api.event.AlterMetalakeFailureEvent; import org.apache.gravitino.listener.api.event.CreateMetalakeEvent; @@ -129,9 +131,10 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) } @Override - public boolean dropMetalake(NameIdentifier ident) { + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { try { - boolean isExists = dispatcher.dropMetalake(ident); + boolean isExists = dispatcher.dropMetalake(ident, force); eventBus.dispatchEvent( new DropMetalakeEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); return isExists; @@ -141,4 +144,16 @@ public boolean dropMetalake(NameIdentifier ident) { throw e; } } + + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + // todo: support activate metalake event + dispatcher.enableMetalake(ident); + } + + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + // todo: support deactivate metalake event + dispatcher.disableMetalake(ident); + } } diff --git a/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java b/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java index bab042d378f..4e2ca045f2e 100644 --- a/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java +++ b/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java @@ -25,13 +25,13 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.apache.gravitino.Audit; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; import org.apache.gravitino.Field; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.Metalake; -import org.apache.gravitino.StringIdentifier; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.metalake.MetalakePropertiesMetadata; /** Base implementation of a Metalake entity. */ @EqualsAndHashCode @@ -52,6 +52,8 @@ public class BaseMetalake implements Metalake, Entity, Auditable, HasIdentifier public static final Field SCHEMA_VERSION = Field.required("version", SchemaVersion.class, "The version of the schema for the metalake"); + public static final PropertiesMetadata PROPERTIES_METADATA = new MetalakePropertiesMetadata(); + private Long id; private String name; @@ -87,10 +89,10 @@ public Map fields() { /** * The audit information of the metalake. * - * @return The audit information as an {@link Audit} instance. + * @return The audit information as an {@link AuditInfo} instance. */ @Override - public Audit auditInfo() { + public AuditInfo auditInfo() { return auditInfo; } @@ -141,7 +143,11 @@ public EntityType type() { */ @Override public Map properties() { - return StringIdentifier.newPropertiesWithoutId(properties); + return properties; + } + + public PropertiesMetadata propertiesMetadata() { + return PROPERTIES_METADATA; } /** Builder class for creating instances of {@link BaseMetalake}. */ diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java index 54298c0efdd..8393a83e450 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java @@ -18,9 +18,12 @@ */ package org.apache.gravitino.metalake; +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; + import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.Map; import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; @@ -28,13 +31,16 @@ import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; -import org.apache.gravitino.StringIdentifier; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.BaseMetalake; +import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.meta.SchemaVersion; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.utils.PrincipalUtils; @@ -63,6 +69,48 @@ public MetalakeManager(EntityStore store, IdGenerator idGenerator) { this.idGenerator = idGenerator; } + /** + * Check whether the metalake is available + * + * @param ident The identifier of the Metalake to check. + * @param store The EntityStore to use for managing Metalakes. + * @throws NoSuchMetalakeException If the Metalake with the given identifier does not exist. + * @throws MetalakeNotInUseException If the Metalake is not in use. + */ + public static void checkMetalake(NameIdentifier ident, EntityStore store) + throws NoSuchMetalakeException, MetalakeNotInUseException { + boolean metalakeInUse = metalakeInUse(store, ident); + if (!metalakeInUse) { + throw new MetalakeNotInUseException( + "Metalake %s is not in use, please activate it first", ident); + } + } + + /** + * Return true if the metalake is in used, false otherwise. + * + * @param store The EntityStore to use for managing Metalakes. + * @param ident The identifier of the Metalake to check. + * @return True if the metalake is in use, false otherwise. + * @throws NoSuchMetalakeException If the Metalake with the given identifier does not exist. + */ + public static boolean metalakeInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException { + try { + BaseMetalake metalake = store.get(ident, EntityType.METALAKE, BaseMetalake.class); + return (boolean) + metalake.propertiesMetadata().getOrDefault(metalake.properties(), PROPERTY_IN_USE); + + } catch (NoSuchEntityException e) { + LOG.warn("Metalake {} does not exist", ident, e); + throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, ident); + + } catch (IOException e) { + LOG.error("Failed to do store operation", e); + throw new RuntimeException(e); + } + } + /** * Lists all available Metalakes. * @@ -72,8 +120,9 @@ public MetalakeManager(EntityStore store, IdGenerator idGenerator) { @Override public BaseMetalake[] listMetalakes() { try { - return store.list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE).stream() - .toArray(BaseMetalake[]::new); + return store + .list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE) + .toArray(new BaseMetalake[0]); } catch (IOException ioe) { LOG.error("Listing Metalakes failed due to storage issues.", ioe); throw new RuntimeException(ioe); @@ -116,14 +165,13 @@ public BaseMetalake createMetalake( NameIdentifier ident, String comment, Map properties) throws MetalakeAlreadyExistsException { long uid = idGenerator.nextId(); - StringIdentifier stringId = StringIdentifier.fromId(uid); BaseMetalake metalake = BaseMetalake.builder() .withId(uid) .withName(ident.name()) .withComment(comment) - .withProperties(StringIdentifier.newPropertiesWithId(stringId, properties)) + .withProperties(properties) .withVersion(SchemaVersion.V_0_1) .withAuditInfo( AuditInfo.builder() @@ -158,28 +206,17 @@ public BaseMetalake createMetalake( public BaseMetalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException { try { + if (!metalakeInUse(store, ident)) { + throw new MetalakeNotInUseException( + "Metalake %s is not in use, please activate it first", ident); + } + return store.update( ident, BaseMetalake.class, EntityType.METALAKE, metalake -> { - BaseMetalake.Builder builder = - BaseMetalake.builder() - .withId(metalake.id()) - .withName(metalake.name()) - .withComment(metalake.comment()) - .withProperties(metalake.properties()) - .withVersion(metalake.getVersion()); - - AuditInfo newInfo = - AuditInfo.builder() - .withCreator(metalake.auditInfo().creator()) - .withCreateTime(metalake.auditInfo().createTime()) - .withLastModifier( - metalake.auditInfo().creator()) /*TODO: Use real user later on. */ - .withLastModifiedTime(Instant.now()) - .build(); - builder.withAuditInfo(newInfo); + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); Map newProps = metalake.properties() == null @@ -204,23 +241,112 @@ public BaseMetalake alterMetalake(NameIdentifier ident, MetalakeChange... change } } - /** - * Deletes a Metalake. - * - * @param ident The identifier of the Metalake to be deleted. - * @return `true` if the Metalake was successfully deleted, `false` otherwise. - * @throws RuntimeException If deleting the Metalake encounters storage issues. - */ @Override - public boolean dropMetalake(NameIdentifier ident) { + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { try { - return store.delete(ident, EntityType.METALAKE); - } catch (IOException ioe) { - LOG.error("Deleting metalake {} failed due to storage issues", ident, ioe); - throw new RuntimeException(ioe); + boolean inUse = metalakeInUse(store, ident); + if (inUse && !force) { + throw new MetalakeInUseException( + "Metalake %s is in use, please deactivate it first or use force option", ident); + } + + List catalogEntities = + store.list(Namespace.of(ident.name()), CatalogEntity.class, EntityType.CATALOG); + if (!catalogEntities.isEmpty() && !force) { + throw new NonEmptyEntityException( + "Metalake %s has catalogs, please drop them first or use force option", ident); + } + + // If reached here, it implies that the metalake is not in use or force is true. + if (inUse) { + // force is true, so deactivate the metalake first. + disableMetalake(ident); + } + + return store.delete(ident, EntityType.METALAKE, true); + } catch (NoSuchMetalakeException e) { + return false; + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + try { + + boolean inUse = metalakeInUse(store, ident); + if (!inUse) { + store.update( + ident, + BaseMetalake.class, + EntityType.METALAKE, + metalake -> { + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); + + Map newProps = + metalake.properties() == null + ? Maps.newHashMap() + : Maps.newHashMap(metalake.properties()); + newProps.put(PROPERTY_IN_USE, "true"); + builder.withProperties(newProps); + + return builder.build(); + }); + } + } catch (IOException e) { + throw new RuntimeException(e); } } + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + try { + boolean inUse = metalakeInUse(store, ident); + if (inUse) { + store.update( + ident, + BaseMetalake.class, + EntityType.METALAKE, + metalake -> { + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); + + Map newProps = + metalake.properties() == null + ? Maps.newHashMap() + : Maps.newHashMap(metalake.properties()); + newProps.put(PROPERTY_IN_USE, "false"); + builder.withProperties(newProps); + + return builder.build(); + }); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private BaseMetalake.Builder newMetalakeBuilder(BaseMetalake metalake) { + BaseMetalake.Builder builder = + BaseMetalake.builder() + .withId(metalake.id()) + .withName(metalake.name()) + .withComment(metalake.comment()) + .withProperties(metalake.properties()) + .withVersion(metalake.getVersion()); + + AuditInfo newInfo = + AuditInfo.builder() + .withCreator(metalake.auditInfo().creator()) + .withCreateTime(metalake.auditInfo().createTime()) + .withLastModifier(metalake.auditInfo().creator()) /*TODO: Use real user later on. */ + .withLastModifiedTime(Instant.now()) + .build(); + return builder.withAuditInfo(newInfo); + } + /** * Updates an entity with the provided changes. * diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java index b0c8cbf1766..dbc9d6bdc29 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java @@ -19,16 +19,25 @@ package org.apache.gravitino.metalake; import static org.apache.gravitino.Entity.SYSTEM_METALAKE_RESERVED_NAME; +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import static org.apache.gravitino.meta.BaseMetalake.PROPERTIES_METADATA; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import java.util.Arrays; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Metalake; import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; +import org.apache.gravitino.meta.BaseMetalake; public class MetalakeNormalizeDispatcher implements MetalakeDispatcher { private static final Set RESERVED_WORDS = ImmutableSet.of(SYSTEM_METALAKE_RESERVED_NAME); @@ -57,7 +66,7 @@ public Metalake[] listMetalakes() { @Override public Metalake loadMetalake(NameIdentifier ident) throws NoSuchMetalakeException { - return dispatcher.loadMetalake(ident); + return newMetalakeWithResolvedProperties((BaseMetalake) dispatcher.loadMetalake(ident)); } @Override @@ -70,6 +79,7 @@ public Metalake createMetalake( NameIdentifier ident, String comment, Map properties) throws MetalakeAlreadyExistsException { validateMetalakeName(ident.name()); + validatePropertyForCreate(PROPERTIES_METADATA, properties); return dispatcher.createMetalake(ident, comment, properties); } @@ -83,6 +93,10 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) validateMetalakeName(((MetalakeChange.RenameMetalake) change).getNewName()); } }); + Pair, Map> alterProperty = + getMetalakeAlterProperty(changes); + validatePropertyForAlter( + PROPERTIES_METADATA, alterProperty.getLeft(), alterProperty.getRight()); return dispatcher.alterMetalake(ident, changes); } @@ -93,6 +107,22 @@ public boolean dropMetalake(NameIdentifier ident) { return dispatcher.dropMetalake(ident); } + @Override + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { + return dispatcher.dropMetalake(ident, force); + } + + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.enableMetalake(ident); + } + + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.disableMetalake(ident); + } + private void validateMetalakeName(String name) { if (RESERVED_WORDS.contains(name)) { throw new IllegalArgumentException("The metalake name '" + name + "' is reserved."); @@ -101,4 +131,45 @@ private void validateMetalakeName(String name) { throw new IllegalArgumentException("The metalake name '" + name + "' is illegal."); } } + + private BaseMetalake newMetalakeWithResolvedProperties(BaseMetalake metalakeEntity) { + Map newProps = Maps.newHashMap(metalakeEntity.properties()); + newProps + .entrySet() + .removeIf(e -> metalakeEntity.propertiesMetadata().isHiddenProperty(e.getKey())); + newProps.putIfAbsent( + PROPERTY_IN_USE, + metalakeEntity.propertiesMetadata().getDefaultValue(PROPERTY_IN_USE).toString()); + + return BaseMetalake.builder() + .withId(metalakeEntity.id()) + .withName(metalakeEntity.name()) + .withComment(metalakeEntity.comment()) + .withProperties(newProps) + .withVersion(metalakeEntity.getVersion()) + .withAuditInfo(metalakeEntity.auditInfo()) + .build(); + } + + private Pair, Map> getMetalakeAlterProperty( + MetalakeChange... metalakeChanges) { + Map upserts = Maps.newHashMap(); + Map deletes = Maps.newHashMap(); + + Arrays.stream(metalakeChanges) + .forEach( + metalakeChange -> { + if (metalakeChange instanceof MetalakeChange.SetProperty) { + MetalakeChange.SetProperty setProperty = + (MetalakeChange.SetProperty) metalakeChange; + upserts.put(setProperty.getProperty(), setProperty.getValue()); + } else if (metalakeChange instanceof MetalakeChange.RemoveProperty) { + MetalakeChange.RemoveProperty removeProperty = + (MetalakeChange.RemoveProperty) metalakeChange; + deletes.put(removeProperty.getProperty(), removeProperty.getProperty()); + } + }); + + return Pair.of(upserts, deletes); + } } diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java new file mode 100644 index 00000000000..7a39b0b63f8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.metalake; + +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; + +public class MetalakePropertiesMetadata implements PropertiesMetadata { + + private static final Map> PROPERTY_ENTRIES; + + static { + List> propertyEntries = + ImmutableList.of( + PropertyEntry.booleanReservedPropertyEntry( + PROPERTY_IN_USE, + "The property indicating the catalog is in use", + true /* default value */, + false /* hidden */)); + + PROPERTY_ENTRIES = Maps.uniqueIndex(propertyEntries, PropertyEntry::getName); + } + + @Override + public Map> propertyEntries() { + return PROPERTY_ENTRIES; + } +} diff --git a/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java b/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java index bd00c7ff5e8..481bfe84476 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java +++ b/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java @@ -24,7 +24,10 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Interface for supporting metalakes. It includes methods for listing, loading, creating, altering @@ -89,10 +92,68 @@ Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException; /** - * Drop a metalake with specified identifier. + * Drop a metalake with specified identifier. Please make sure: + * + *
    + *
  • There is no catalog in the metalake. Otherwise, a {@link NonEmptyEntityException} will be + * thrown. + *
  • The method {@link #disableMetalake(NameIdentifier)} has been called before dropping the + * metalake. Otherwise, a {@link MetalakeInUseException} will be thrown. + *
+ * + * It is equivalent to calling {@code dropMetalake(ident, false)}. + * + * @param ident The identifier of the metalake. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty. + * @throws MetalakeInUseException If the metalake is in use. + */ + default boolean dropMetalake(NameIdentifier ident) + throws NonEmptyEntityException, MetalakeInUseException { + return dropMetalake(ident, false); + } + + /** + * Drop a metalake with specified identifier. If the force flag is true, it will: + * + *
    + *
  • Cascade drop all sub-entities (tags, catalogs, schemas, tables, etc.) of the metalake in + * Gravitino store. + *
  • Drop the metalake even if it is in use. + *
  • External resources (e.g. database, table, etc.) associated with sub-entities will not be + * deleted unless it is managed (such as managed fileset). + *
* * @param ident The identifier of the metalake. + * @param force Whether to force the drop. * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws MetalakeInUseException If the metalake is in use and force is false. + */ + boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException; + + /** + * Enable a metalake. If the metalake is already in use, this method does nothing. + * + * @param ident The identifier of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. + */ + void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException; + + /** + * Disable a metalake. If the metalake is already disabled, this method does nothing. Once a + * metalake is disable: + * + *
    + *
  • It can only be listed, loaded, dropped, or enable. + *
  • Any other operations on the metalake will throw an {@link MetalakeNotInUseException}. + *
  • Any operation on the sub-entities (catalogs, schemas, tables, etc.) will throw an {@link + * MetalakeNotInUseException}. + *
+ * + * @param ident The identifier of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. */ - boolean dropMetalake(NameIdentifier ident); + void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException; } diff --git a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java b/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java index d8536fffbdb..e18b280e691 100644 --- a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java +++ b/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java @@ -20,7 +20,6 @@ package org.apache.gravitino.proto; import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.AuditInfo; /** A class for serializing and deserializing BaseMetalake objects. */ class BaseMetalakeSerDe implements ProtoSerDe { @@ -38,7 +37,7 @@ public Metalake serialize(org.apache.gravitino.meta.BaseMetalake baseMetalake) { Metalake.newBuilder() .setId(baseMetalake.id()) .setName(baseMetalake.name()) - .setAuditInfo(new AuditInfoSerDe().serialize((AuditInfo) baseMetalake.auditInfo())); + .setAuditInfo(new AuditInfoSerDe().serialize(baseMetalake.auditInfo())); if (baseMetalake.comment() != null) { builder.setComment(baseMetalake.comment()); diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index aaffd35b50a..1b1626de00e 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.tag; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -35,7 +37,6 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchTagException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; @@ -93,7 +94,7 @@ public Tag[] listTagsInfo(String metalake) { NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore @@ -114,7 +115,7 @@ public Tag createTag(String metalake, String name, String comment, Map { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); TagEntity tagEntity = TagEntity.builder() @@ -148,7 +149,7 @@ public Tag getTag(String metalake, String name) throws NoSuchTagException { ofTagIdent(metalake, name), LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.get( @@ -169,7 +170,7 @@ public Tag alterTag(String metalake, String name, TagChange... changes) NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.WRITE, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.update( @@ -195,7 +196,7 @@ public boolean deleteTag(String metalake, String name) { NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.WRITE, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.delete(ofTagIdent(metalake, name), Entity.EntityType.TAG); @@ -213,7 +214,7 @@ public MetadataObject[] listMetadataObjectsForTag(String metalake, String name) tagId, LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { if (!entityStore.exists(tagId, Entity.EntityType.TAG)) { @@ -356,19 +357,6 @@ public String[] associateTagsForMetadataObject( })); } - private static void checkMetalakeExists(String metalake, EntityStore entityStore) { - try { - NameIdentifier metalakeIdent = NameIdentifier.of(metalake); - if (!entityStore.exists(metalakeIdent, Entity.EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException("Metalake %s does not exist", metalakeIdent); - } - } catch (IOException ioe) { - LOG.error("Failed to check if metalake exists", ioe); - throw new RuntimeException(ioe); - } - } - public static Namespace ofTagNamespace(String metalake) { return Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.TAG_SCHEMA_NAME); } diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java index 319ac641f61..2b5377f7598 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java @@ -19,6 +19,7 @@ package org.apache.gravitino.listener.api.event; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -205,7 +206,7 @@ private MetalakeDispatcher mockMetalakeDispatcher() { when(dispatcher.createMetalake(any(NameIdentifier.class), any(String.class), any(Map.class))) .thenReturn(metalake); when(dispatcher.loadMetalake(any(NameIdentifier.class))).thenReturn(metalake); - when(dispatcher.dropMetalake(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.dropMetalake(any(NameIdentifier.class), anyBoolean())).thenReturn(true); when(dispatcher.listMetalakes()).thenReturn(null); when(dispatcher.alterMetalake(any(NameIdentifier.class), any(MetalakeChange.class))) .thenReturn(metalake); diff --git a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java index 96bc1ebdd82..3b0f796e979 100644 --- a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java +++ b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java @@ -173,6 +173,7 @@ public void testDropMetalake() { Assertions.assertEquals("comment", metalake.comment()); testProperties(props, metalake.properties()); + metalakeManager.disableMetalake(ident); boolean dropped = metalakeManager.dropMetalake(ident); Assertions.assertTrue(dropped, "metalake should be dropped"); diff --git a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java index 6502b6931ea..666ba9aaead 100644 --- a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java @@ -695,9 +695,7 @@ void testEntityDelete(String type) throws IOException { // metalake BaseMetalake metalakeNew = createBaseMakeLake( - RandomIdGenerator.INSTANCE.nextId(), - metalake.name(), - (AuditInfo) metalake.auditInfo()); + RandomIdGenerator.INSTANCE.nextId(), metalake.name(), metalake.auditInfo()); store.put(metalakeNew); // catalog CatalogEntity catalogNew = @@ -976,7 +974,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake1"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake1New.id(), "metalake2", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake1New.id(), "metalake2", e.auditInfo())); // Rename metalake3 --> metalake1 BaseMetalake metalake3New1 = @@ -986,7 +984,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake3"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake3New1.id(), "metalake1", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake3New1.id(), "metalake1", e.auditInfo())); // Rename metalake3 --> metalake2 BaseMetalake metalake3New2 = @@ -998,7 +996,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake3"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake3New2.id(), "metalake2", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake3New2.id(), "metalake2", e.auditInfo())); // Finally, only metalake2 and metalake1 are left. Assertions.assertDoesNotThrow( diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index 2fe844d8c02..284a07b849f 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -24,16 +24,18 @@ import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; -import org.apache.gravitino.exceptions.CatalogInUseException; -import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.ConnectionFailedException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.ForbiddenException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.InUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NonEmptySchemaException; import org.apache.gravitino.exceptions.NotFoundException; +import org.apache.gravitino.exceptions.NotInUseException; import org.apache.gravitino.exceptions.PartitionAlreadyExistsException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; @@ -131,6 +133,9 @@ public static Response handleTestConnectionException(Exception e) { } else if (e instanceof AlreadyExistsException) { response = ErrorResponse.alreadyExists(e.getClass().getSimpleName(), e.getMessage(), e); + } else if (e instanceof NotInUseException) { + response = ErrorResponse.notInUse(e.getClass().getSimpleName(), e.getMessage(), e); + } else { return Utils.internalError(e.getMessage(), e); } @@ -180,7 +185,7 @@ public Response handle(OperationType op, String partition, String table, Excepti } else if (e instanceof UnsupportedOperationException) { return Utils.unsupportedOperation(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -221,7 +226,7 @@ public Response handle(OperationType op, String table, String schema, Exception } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -265,7 +270,7 @@ public Response handle(OperationType op, String schema, String catalog, Exceptio } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -306,10 +311,10 @@ public Response handle(OperationType op, String catalog, String metalake, Except } else if (e instanceof CatalogAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); - } else if (e instanceof CatalogInUseException) { + } else if (e instanceof InUseException) { return Utils.inUse(errorMsg, e); } else { @@ -343,6 +348,12 @@ public Response handle(OperationType op, String metalake, String parent, Excepti } else if (e instanceof NoSuchMetalakeException) { return Utils.notFound(errorMsg, e); + } else if (e instanceof MetalakeNotInUseException) { + return Utils.notInUse(errorMsg, e); + + } else if (e instanceof MetalakeInUseException) { + return Utils.inUse(errorMsg, e); + } else { return super.handle(op, metalake, parent, e); } @@ -377,7 +388,7 @@ public Response handle(OperationType op, String fileset, String schema, Exceptio } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -418,6 +429,9 @@ public Response handle(OperationType op, String user, String metalake, Exception } else if (e instanceof UserAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, user, metalake, e); } @@ -450,6 +464,9 @@ public Response handle(OperationType op, String group, String metalake, Exceptio } else if (e instanceof GroupAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, group, metalake, e); } @@ -485,6 +502,9 @@ public Response handle(OperationType op, String role, String metalake, Exception } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, role, metalake, e); } @@ -518,6 +538,10 @@ public Response handle(OperationType op, String topic, String schema, Exception } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, topic, schema, e); } @@ -581,6 +605,9 @@ public Response handle(OperationType op, String roles, String parent, Exception } else if (e instanceof NotFoundException) { return Utils.notFound(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, roles, parent, e); } @@ -616,6 +643,9 @@ public Response handle(OperationType op, String tag, String parent, Exception e) } else if (e instanceof TagAlreadyAssociatedException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, tag, parent, e); } @@ -643,6 +673,10 @@ public Response handle(OperationType op, String name, String parent, Exception e } else if (e instanceof NotFoundException) { return Utils.notFound(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, name, parent, e); } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java index fff86cf2e87..28bd2d339e5 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java @@ -25,12 +25,15 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.PATCH; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -40,8 +43,10 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.dto.MetalakeDTO; import org.apache.gravitino.dto.requests.MetalakeCreateRequest; +import org.apache.gravitino.dto.requests.MetalakeSetRequest; import org.apache.gravitino.dto.requests.MetalakeUpdateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdatesRequest; +import org.apache.gravitino.dto.responses.BaseResponse; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.MetalakeListResponse; import org.apache.gravitino.dto.responses.MetalakeResponse; @@ -149,6 +154,101 @@ public Response loadMetalake(@PathParam("name") String metalakeName) { } } + @PATCH + @Path("{name}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "set-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "set-metalake", absolute = true) + public Response setMetalake(@PathParam("name") String metalakeName, MetalakeSetRequest request) { + LOG.info("Received set request for metalake: {}", metalakeName); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); + TreeLockUtils.doWithTreeLock( + identifier, + LockType.WRITE, + () -> { + if (request.isInUse()) { + metalakeDispatcher.enableMetalake(identifier); + } else { + metalakeDispatcher.disableMetalake(identifier); + } + return null; + }); + Response response = Utils.ok(new BaseResponse()); + LOG.info( + "Successfully {} metalake: {}", + request.isInUse() ? "enable" : "disable", + metalakeName); + return response; + }); + + } catch (Exception e) { + LOG.info("Failed to {} metalake: {}", request.isInUse() ? "enable" : "disable", metalakeName); + return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); + } + } + + @GET + @Path("{name}/activate") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "activate-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "activate-metalake", absolute = true) + public Response activateMetalake(@PathParam("name") String metalakeName) { + LOG.info("Received activate request for metalake: {}", metalakeName); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); + TreeLockUtils.doWithTreeLock( + identifier, + LockType.READ, + () -> { + metalakeDispatcher.enableMetalake(identifier); + return null; + }); + Response response = Utils.ok(new BaseResponse()); + LOG.info("Metalake activated: {}", metalakeName); + return response; + }); + + } catch (Exception e) { + return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); + } + } + + @GET + @Path("{name}/deactivate") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "deactivate-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "deactivate-metalake", absolute = true) + public Response deactivateMetalake(@PathParam("name") String metalakeName) { + LOG.info("Received deactivate request for metalake: {}", metalakeName); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); + TreeLockUtils.doWithTreeLock( + identifier, + LockType.READ, + () -> { + metalakeDispatcher.disableMetalake(identifier); + return null; + }); + Response response = Utils.ok(new BaseResponse()); + LOG.info("Metalake deactivated: {}", metalakeName); + return response; + }); + + } catch (Exception e) { + return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); + } + } + @PUT @Path("{name}") @Produces("application/vnd.gravitino.v1+json") @@ -186,7 +286,9 @@ public Response alterMetalake( @Produces("application/vnd.gravitino.v1+json") @Timed(name = "drop-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "drop-metalake", absolute = true) - public Response dropMetalake(@PathParam("name") String metalakeName) { + public Response dropMetalake( + @PathParam("name") String metalakeName, + @DefaultValue("false") @QueryParam("force") boolean force) { LOG.info("Received drop metalake request for metalake: {}", metalakeName); try { return Utils.doAs( @@ -195,7 +297,7 @@ public Response dropMetalake(@PathParam("name") String metalakeName) { NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); boolean dropped = TreeLockUtils.doWithRootTreeLock( - LockType.WRITE, () -> metalakeDispatcher.dropMetalake(identifier)); + LockType.WRITE, () -> metalakeDispatcher.dropMetalake(identifier, force)); if (!dropped) { LOG.warn("Failed to drop metalake by name {}", metalakeName); } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java index 3b73cf035be..9b265e016d2 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java @@ -22,6 +22,7 @@ import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -381,7 +382,7 @@ public void testAlterMetalake() { @Test public void testDropMetalake() { - when(metalakeManager.dropMetalake(any())).thenReturn(true); + when(metalakeManager.dropMetalake(any(), anyBoolean())).thenReturn(true); Response resp = target("/metalakes/test") .request(MediaType.APPLICATION_JSON_TYPE) @@ -396,7 +397,7 @@ public void testDropMetalake() { Assertions.assertTrue(dropped); // Test when failed to drop metalake - when(metalakeManager.dropMetalake(any())).thenReturn(false); + when(metalakeManager.dropMetalake(any(), anyBoolean())).thenReturn(false); Response resp2 = target("/metalakes/test") .request(MediaType.APPLICATION_JSON_TYPE) @@ -408,7 +409,9 @@ public void testDropMetalake() { Assertions.assertFalse(dropResponse2.dropped()); // Test throw an exception when deleting tenant. - doThrow(new RuntimeException("Internal error")).when(metalakeManager).dropMetalake(any()); + doThrow(new RuntimeException("Internal error")) + .when(metalakeManager) + .dropMetalake(any(), anyBoolean()); Response resp1 = target("/metalakes/test") diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java index eabd7732194..5271197d2d3 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java @@ -156,7 +156,7 @@ private static void dropMetalake() { if (!exists) { return; } - gravitinoClient.dropMetalake(metalakeName); + gravitinoClient.dropMetalake(metalakeName, true); } private static void createCatalog( diff --git a/web/web/src/lib/api/metalakes/index.js b/web/web/src/lib/api/metalakes/index.js index a34fad5a4bf..5b7b142284b 100644 --- a/web/web/src/lib/api/metalakes/index.js +++ b/web/web/src/lib/api/metalakes/index.js @@ -47,7 +47,7 @@ export const createMetalakeApi = data => { export const deleteMetalakeApi = name => { return defHttp.delete({ - url: `${Apis.DELETE}/${name}` + url: `${Apis.DELETE}/${name}?force=true` }) } From 72d5db936512d458985de67ae6829c27317fdaf2 Mon Sep 17 00:00:00 2001 From: mchades Date: Sun, 20 Oct 2024 22:59:45 +0800 Subject: [PATCH 2/3] address comments --- .../client/GravitinoAdminClient.java | 15 +++-- .../gravitino/catalog/CatalogManager.java | 6 +- .../listener/CatalogEventDispatcher.java | 2 +- .../listener/MetalakeEventDispatcher.java | 4 +- .../gravitino/metalake/MetalakeManager.java | 12 +--- .../server/web/rest/MetalakeOperations.java | 58 ------------------- 6 files changed, 20 insertions(+), 77 deletions(-) diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java index 32c425fc76a..7a4530235c9 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java @@ -29,6 +29,7 @@ import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.SupportsMetalakes; import org.apache.gravitino.dto.requests.MetalakeCreateRequest; +import org.apache.gravitino.dto.requests.MetalakeSetRequest; import org.apache.gravitino.dto.requests.MetalakeUpdateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdatesRequest; import org.apache.gravitino.dto.responses.DropResponse; @@ -187,9 +188,12 @@ public boolean dropMetalake(String name, boolean force) @Override public void enableMetalake(String name) throws NoSuchMetalakeException { + MetalakeSetRequest req = new MetalakeSetRequest(true); + ErrorResponse resp = - restClient.get( - API_METALAKES_IDENTIFIER_PATH + name + "/activate", + restClient.patch( + API_METALAKES_IDENTIFIER_PATH + name, + req, ErrorResponse.class, Collections.emptyMap(), ErrorHandlers.metalakeErrorHandler()); @@ -203,9 +207,12 @@ public void enableMetalake(String name) throws NoSuchMetalakeException { @Override public void disableMetalake(String name) throws NoSuchMetalakeException { + MetalakeSetRequest req = new MetalakeSetRequest(false); + ErrorResponse resp = - restClient.get( - API_METALAKES_IDENTIFIER_PATH + name + "/deactivate", + restClient.patch( + API_METALAKES_IDENTIFIER_PATH + name, + req, ErrorResponse.class, Collections.emptyMap(), ErrorHandlers.metalakeErrorHandler()); diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index aed3a676e4f..6af759c6142 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -116,7 +116,7 @@ public static void checkCatalogInUse(EntityStore store, NameIdentifier ident) NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); checkMetalake(metalakeIdent, store); - if (!getInUseValue(store, ident)) { + if (!getCatalogInUseValue(store, ident)) { throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); } } @@ -677,10 +677,10 @@ public CatalogWrapper loadCatalogAndWrap(NameIdentifier ident) throws NoSuchCata private static boolean catalogInUse(EntityStore store, NameIdentifier ident) throws NoSuchMetalakeException, NoSuchCatalogException { NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); - return metalakeInUse(store, metalakeIdent) && getInUseValue(store, ident); + return metalakeInUse(store, metalakeIdent) && getCatalogInUseValue(store, ident); } - private static boolean getInUseValue(EntityStore store, NameIdentifier catalogIdent) { + private static boolean getCatalogInUseValue(EntityStore store, NameIdentifier catalogIdent) { try { CatalogEntity catalogEntity = store.get(catalogIdent, EntityType.CATALOG, CatalogEntity.class); diff --git a/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java index 04a2600d8ec..ff70d4cffda 100644 --- a/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java @@ -177,7 +177,7 @@ public void testConnection( @Override public void enableCatalog(NameIdentifier ident) throws NoSuchCatalogException, CatalogNotInUseException { - // todo: support activate catalog event + // todo: support enable catalog event dispatcher.enableCatalog(ident); } diff --git a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java index 23f9b328b47..535d337b303 100644 --- a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java @@ -147,13 +147,13 @@ public boolean dropMetalake(NameIdentifier ident, boolean force) @Override public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { - // todo: support activate metalake event + // todo: support enable metalake event dispatcher.enableMetalake(ident); } @Override public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { - // todo: support deactivate metalake event + // todo: support disable metalake event dispatcher.disableMetalake(ident); } } diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java index 8393a83e450..33498665a2d 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java @@ -82,7 +82,7 @@ public static void checkMetalake(NameIdentifier ident, EntityStore store) boolean metalakeInUse = metalakeInUse(store, ident); if (!metalakeInUse) { throw new MetalakeNotInUseException( - "Metalake %s is not in use, please activate it first", ident); + "Metalake %s is not in use, please enable it first", ident); } } @@ -208,7 +208,7 @@ public BaseMetalake alterMetalake(NameIdentifier ident, MetalakeChange... change try { if (!metalakeInUse(store, ident)) { throw new MetalakeNotInUseException( - "Metalake %s is not in use, please activate it first", ident); + "Metalake %s is not in use, please enable it first", ident); } return store.update( @@ -248,7 +248,7 @@ public boolean dropMetalake(NameIdentifier ident, boolean force) boolean inUse = metalakeInUse(store, ident); if (inUse && !force) { throw new MetalakeInUseException( - "Metalake %s is in use, please deactivate it first or use force option", ident); + "Metalake %s is in use, please disable it first or use force option", ident); } List catalogEntities = @@ -258,12 +258,6 @@ public boolean dropMetalake(NameIdentifier ident, boolean force) "Metalake %s has catalogs, please drop them first or use force option", ident); } - // If reached here, it implies that the metalake is not in use or force is true. - if (inUse) { - // force is true, so deactivate the metalake first. - disableMetalake(ident); - } - return store.delete(ident, EntityType.METALAKE, true); } catch (NoSuchMetalakeException e) { return false; diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java index 28bd2d339e5..e28f28f15e5 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java @@ -191,64 +191,6 @@ public Response setMetalake(@PathParam("name") String metalakeName, MetalakeSetR } } - @GET - @Path("{name}/activate") - @Produces("application/vnd.gravitino.v1+json") - @Timed(name = "activate-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) - @ResponseMetered(name = "activate-metalake", absolute = true) - public Response activateMetalake(@PathParam("name") String metalakeName) { - LOG.info("Received activate request for metalake: {}", metalakeName); - try { - return Utils.doAs( - httpRequest, - () -> { - NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); - TreeLockUtils.doWithTreeLock( - identifier, - LockType.READ, - () -> { - metalakeDispatcher.enableMetalake(identifier); - return null; - }); - Response response = Utils.ok(new BaseResponse()); - LOG.info("Metalake activated: {}", metalakeName); - return response; - }); - - } catch (Exception e) { - return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); - } - } - - @GET - @Path("{name}/deactivate") - @Produces("application/vnd.gravitino.v1+json") - @Timed(name = "deactivate-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) - @ResponseMetered(name = "deactivate-metalake", absolute = true) - public Response deactivateMetalake(@PathParam("name") String metalakeName) { - LOG.info("Received deactivate request for metalake: {}", metalakeName); - try { - return Utils.doAs( - httpRequest, - () -> { - NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); - TreeLockUtils.doWithTreeLock( - identifier, - LockType.READ, - () -> { - metalakeDispatcher.disableMetalake(identifier); - return null; - }); - Response response = Utils.ok(new BaseResponse()); - LOG.info("Metalake deactivated: {}", metalakeName); - return response; - }); - - } catch (Exception e) { - return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); - } - } - @PUT @Path("{name}") @Produces("application/vnd.gravitino.v1+json") From 39572d0eb9ed3263768f8e5fc007df062f94193b Mon Sep 17 00:00:00 2001 From: mchades Date: Mon, 21 Oct 2024 14:47:13 +0800 Subject: [PATCH 3/3] fix comment --- api/src/main/java/org/apache/gravitino/SupportsMetalakes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java index 0508f5239a6..515b2d836cf 100644 --- a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java +++ b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java @@ -98,7 +98,7 @@ Metalake alterMetalake(String name, MetalakeChange... changes) * Otherwise, a {@link MetalakeInUseException} will be thrown. * * - * It is equivalent to calling {@code dropMetalake(ident, false)}. + * It is equivalent to calling {@code dropMetalake(name, false)}. * * @param name The name of the metalake. * @return True if the metalake was dropped, false if the metalake does not exist.