diff --git a/api/src/main/java/org/apache/gravitino/Catalog.java b/api/src/main/java/org/apache/gravitino/Catalog.java index 431a798d51d..343ae54349b 100644 --- a/api/src/main/java/org/apache/gravitino/Catalog.java +++ b/api/src/main/java/org/apache/gravitino/Catalog.java @@ -98,6 +98,9 @@ enum CloudName { */ String PROPERTY_PACKAGE = "package"; + /** The property indicates the catalog is in use. */ + String PROPERTY_IN_USE = "in-use"; + /** * The property to specify the cloud that the catalog is running on. The value should be one of * the {@link CloudName}. diff --git a/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java b/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java index 8644430bc90..b10ef401022 100644 --- a/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java +++ b/api/src/main/java/org/apache/gravitino/SupportsCatalogs.java @@ -21,8 +21,11 @@ import java.util.Map; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Client interface for supporting catalogs. It includes methods for listing, loading, creating, @@ -48,9 +51,9 @@ public interface SupportsCatalogs { Catalog[] listCatalogsInfo() throws NoSuchMetalakeException; /** - * Load a catalog by its identifier. + * Load a catalog by its name. * - * @param catalogName the identifier of the catalog. + * @param catalogName the name of the catalog. * @return The catalog. * @throws NoSuchCatalogException If the catalog does not exist. */ @@ -59,7 +62,7 @@ public interface SupportsCatalogs { /** * Check if a catalog exists. * - * @param catalogName The identifier of the catalog. + * @param catalogName The name of the catalog. * @return True if the catalog exists, false otherwise. */ default boolean catalogExists(String catalogName) { @@ -72,7 +75,7 @@ default boolean catalogExists(String catalogName) { } /** - * Create a catalog with specified identifier. + * Create a catalog with specified catalog name, type, provider, comment, and properties. * *

The parameter "provider" is a short name of the catalog, used to tell Gravitino which * catalog should be created. The short name should be the same as the {@link CatalogProvider} @@ -96,9 +99,9 @@ Catalog createCatalog( throws NoSuchMetalakeException, CatalogAlreadyExistsException; /** - * Alter a catalog with specified identifier. + * Alter a catalog with specified catalog name and changes. * - * @param catalogName the identifier of the catalog. + * @param catalogName the name of the catalog. * @param changes the changes to apply to the catalog. * @return The altered catalog. * @throws NoSuchCatalogException If the catalog does not exist. @@ -108,12 +111,71 @@ Catalog alterCatalog(String catalogName, CatalogChange... changes) throws NoSuchCatalogException, IllegalArgumentException; /** - * Drop a catalog with specified identifier. + * Drop a catalog with specified name. Please make sure: + * + *

+ * + * It is equivalent to calling {@code dropCatalog(ident, false)}. * * @param catalogName the name of the catalog. - * @return True if the catalog was dropped, false otherwise. + * @return True if the catalog was dropped, false if the catalog does not exist. + * @throws NonEmptyEntityException If the catalog is not empty. + * @throws CatalogInUseException If the catalog is in use. + */ + default boolean dropCatalog(String catalogName) + throws NonEmptyEntityException, CatalogInUseException { + return dropCatalog(catalogName, false); + } + + /** + * Drop a catalog with specified name. If the force flag is true, it will: + * + * + * + * If the force flag is false, it is equivalent to calling {@link #dropCatalog(String)}. + * + * @param catalogName The identifier of the catalog. + * @param force Whether to force the drop. + * @return True if the catalog was dropped, false if the catalog does not exist. + * @throws NonEmptyEntityException If the catalog is not empty and force is false. + * @throws CatalogInUseException If the catalog is in use and force is false. + */ + boolean dropCatalog(String catalogName, boolean force) + throws NonEmptyEntityException, CatalogInUseException; + + /** + * Enable a catalog. If the catalog is already enabled, this method does nothing. + * + * @param catalogName The identifier of the catalog. + * @throws NoSuchCatalogException If the catalog does not exist. + */ + void enableCatalog(String catalogName) throws NoSuchCatalogException; + + /** + * Disable a catalog. If the catalog is already disabled, this method does nothing. Once a catalog + * is disabled: + * + * + * + * @param catalogName The identifier of the catalog. + * @throws NoSuchCatalogException If the catalog does not exist. */ - boolean dropCatalog(String catalogName); + void disableCatalog(String catalogName) throws NoSuchCatalogException; /** * Test whether the catalog with specified parameters can be connected to before creating it. diff --git a/api/src/main/java/org/apache/gravitino/exceptions/CatalogInUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/CatalogInUseException.java new file mode 100644 index 00000000000..849822beda4 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/CatalogInUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a catalog is in use and cannot be deleted. */ +public class CatalogInUseException extends InUseException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public CatalogInUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/CatalogNotInUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/CatalogNotInUseException.java new file mode 100644 index 00000000000..22d1616039b --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/CatalogNotInUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when operating on a catalog that is not in use. */ +public class CatalogNotInUseException extends NotInUseException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public CatalogNotInUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/InUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/InUseException.java new file mode 100644 index 00000000000..f30abfb9a8e --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/InUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a resource is in use and cannot be deleted. */ +public class InUseException extends GravitinoRuntimeException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public InUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/NotInUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/NotInUseException.java new file mode 100644 index 00000000000..bc32203a294 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/NotInUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when operating on a resource that is not in use. */ +public class NotInUseException extends GravitinoRuntimeException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public NotInUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index 28f9228e893..0f1f4c05abd 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -190,10 +190,7 @@ public void stop() throws IOException { catalog.asSchemas().dropSchema(schema, true); })); Arrays.stream(metalake.listCatalogs()) - .forEach( - (catalogName -> { - metalake.dropCatalog(catalogName); - })); + .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); client.dropMetalake(metalakeName); } if (sparkSession != null) { diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java index 79777f75d64..b0717a90c51 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java @@ -156,6 +156,10 @@ public boolean dropSchema(NameIdentifier ident, boolean cascade) throws NonEmpty UserContext.clearUserContext(ident); return r; + } catch (NoSuchEntityException e) { + LOG.warn("Schema {} does not exist", ident); + return false; + } catch (IOException ioe) { throw new RuntimeException("Failed to delete schema " + ident, ioe); } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index 76d17ff0146..ef8f37187e1 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -87,7 +87,7 @@ public void setup() throws IOException { public void stop() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); client.dropMetalake(metalakeName); if (fileSystem != null) { fileSystem.close(); @@ -160,7 +160,7 @@ void testAlterCatalogLocation() { Assertions.assertEquals(newLocation, modifiedCatalog.properties().get("location")); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } @Test @@ -606,7 +606,7 @@ public void testDropCatalogWithEmptySchema() { filesetCatalog.asSchemas().schemaExists(schemaName), "schema should not be exists"); // Drop the catalog. - dropped = metalake.dropCatalog(catalogName); + dropped = metalake.dropCatalog(catalogName, true); Assertions.assertTrue(dropped, "catalog should be dropped"); Assertions.assertFalse(metalake.catalogExists(catalogName), "catalog should not be exists"); } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java index 0a23ea7d326..d0de2972742 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java @@ -653,7 +653,7 @@ void testUserImpersonation() { catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, filesetName)); catalog.asSchemas().dropSchema(SCHEMA_NAME, true); - gravitinoMetalake.dropCatalog(catalogName); + gravitinoMetalake.dropCatalog(catalogName, true); adminClient.dropMetalake(metalakeName); } } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java index 0acb4a345a1..9e355ed044b 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/TestHiveCatalogOperations.java @@ -22,6 +22,7 @@ import static org.apache.gravitino.Catalog.AUTHORIZATION_PROVIDER; import static org.apache.gravitino.Catalog.CLOUD_NAME; import static org.apache.gravitino.Catalog.CLOUD_REGION_CODE; +import static org.apache.gravitino.Catalog.PROPERTY_IN_USE; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CHECK_INTERVAL_SEC; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS; import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_SIZE; @@ -73,10 +74,11 @@ void testPropertyMeta() { Map> propertyEntryMap = HIVE_PROPERTIES_METADATA.catalogPropertiesMetadata().propertyEntries(); - Assertions.assertEquals(20, propertyEntryMap.size()); + Assertions.assertEquals(21, propertyEntryMap.size()); Assertions.assertTrue(propertyEntryMap.containsKey(METASTORE_URIS)); Assertions.assertTrue(propertyEntryMap.containsKey(Catalog.PROPERTY_PACKAGE)); Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.CATALOG_OPERATION_IMPL)); + Assertions.assertTrue(propertyEntryMap.containsKey(PROPERTY_IN_USE)); Assertions.assertTrue(propertyEntryMap.containsKey(AUTHORIZATION_PROVIDER)); Assertions.assertTrue(propertyEntryMap.containsKey(CLIENT_POOL_SIZE)); Assertions.assertTrue(propertyEntryMap.containsKey(IMPERSONATION_ENABLE)); diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index 1233870b0b1..31493d54ba2 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -228,10 +228,7 @@ public void stop() throws IOException { catalog.asSchemas().dropSchema(schema, true); })); Arrays.stream(metalake.listCatalogs()) - .forEach( - (catalogName -> { - metalake.dropCatalog(catalogName); - })); + .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); client.dropMetalake(metalakeName); } if (hiveClientPool != null) { @@ -1672,7 +1669,7 @@ void testAlterCatalogProperties() { }); newCatalog.asSchemas().dropSchema("schema", true); - metalake.dropCatalog(nameOfCatalog); + metalake.dropCatalog(nameOfCatalog, true); } private void createCatalogWithCustomOperation(String catalogName, String customImpl) { diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java index d6330b0d819..c333cf35103 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java @@ -269,7 +269,7 @@ public void testUserAuthentication() { Assertions.assertFalse(catalog.asSchemas().schemaExists(SCHEMA_NAME)); // Drop catalog - Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME)); + Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME, true)); } @AfterAll diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index a6059a56e74..b08aa491600 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -125,7 +125,7 @@ public void startup() throws IOException { @AfterAll public void stop() { clearTableAndSchema(); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); client.dropMetalake(metalakeName); } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java index a70b7007050..784361d6407 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java @@ -95,7 +95,7 @@ public void testAuditCatalog() throws Exception { Assertions.assertEquals(expectUser, catalog.auditInfo().creator()); Assertions.assertEquals(expectUser, catalog.auditInfo().lastModifier()); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } @Test @@ -109,7 +109,7 @@ public void testAuditSchema() throws Exception { Assertions.assertNull(schema.auditInfo().lastModifier()); catalog.asSchemas().dropSchema(schemaName, true); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } @Test @@ -144,7 +144,7 @@ public void testAuditTable() throws Exception { catalog.asTableCatalog().dropTable(NameIdentifier.of(schemaName, tableName)); catalog.asSchemas().dropSchema(schemaName, true); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } private static Catalog createCatalog(String catalogName) throws SQLException { diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index fe43538e3b5..60af07011e8 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -142,6 +142,7 @@ public void startup() throws IOException, SQLException { @AfterAll public void stop() { clearTableAndSchema(); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); client.dropMetalake(metalakeName); mysqlService.close(); @@ -1987,6 +1988,6 @@ void testAlterCatalogProperties() throws SQLException { Assertions.assertDoesNotThrow(() -> loadCatalog.asSchemas().createSchema("test", "", null)); loadCatalog.asSchemas().dropSchema("test", true); - metalake.dropCatalog(testCatalogName); + metalake.dropCatalog(testCatalogName, true); } } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index c657b04ca74..0d1292c67a2 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -128,6 +128,7 @@ public void stop() { for (String schemaName : schemaNames) { catalog.asSchemas().dropSchema(schemaName, true); } + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); client.dropMetalake(metalakeName); postgreSqlService.close(); diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index baa83619873..907b00733bc 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -123,6 +123,7 @@ public void shutdown() { Arrays.stream(metalake.listCatalogs()) .forEach( (catalogName -> { + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); })); client.dropMetalake(METALAKE_NAME); @@ -171,7 +172,7 @@ public void testCatalog() throws ExecutionException, InterruptedException { Assertions.assertFalse(alteredCatalog.properties().containsKey("key1")); // test drop catalog - boolean dropped = metalake.dropCatalog(catalogName); + boolean dropped = metalake.dropCatalog(catalogName, true); Assertions.assertTrue(dropped); Exception exception = Assertions.assertThrows( diff --git a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java index a3d07125f3b..dcc7e1ad916 100644 --- a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java +++ b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java @@ -18,12 +18,14 @@ */ package org.apache.gravitino.catalog.lakehouse.hudi.integration.test; +import static org.apache.gravitino.Catalog.PROPERTY_IN_USE; import static org.apache.gravitino.catalog.lakehouse.hudi.HudiCatalogPropertiesMetadata.CATALOG_BACKEND; import static org.apache.gravitino.catalog.lakehouse.hudi.HudiCatalogPropertiesMetadata.URI; import static org.apache.gravitino.catalog.lakehouse.hudi.HudiSchemaPropertiesMetadata.LOCATION; import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -146,7 +148,9 @@ public void testCatalog() { Assertions.assertEquals(Catalog.Type.RELATIONAL, catalog.type()); Assertions.assertEquals("lakehouse-hudi", catalog.provider()); Assertions.assertEquals(comment, catalog.comment()); - Assertions.assertEquals(properties, catalog.properties()); + Map expectedProperties = new HashMap<>(properties); + expectedProperties.put(PROPERTY_IN_USE, "true"); + Assertions.assertEquals(expectedProperties, catalog.properties()); // test list String[] catalogs = metalake.listCatalogs(); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index 1c60e04b4c9..9fcc93451ca 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -138,6 +138,7 @@ public void startup() throws Exception { public void stop() throws Exception { try { clearTableAndSchema(); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); client.dropMetalake(metalakeName); } finally { diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java index a9ca1a1108c..b78b7c2489a 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergHiveIT.java @@ -77,6 +77,6 @@ void testAlterCatalogProperties() { () -> createdCatalog.asSchemas().createSchema("schema1", "", null)); createdCatalog.asSchemas().dropSchema("schema1", false); - metalake.dropCatalog(catalogNm); + metalake.dropCatalog(catalogNm, true); } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java index 0f899463203..1017ccb4e10 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java @@ -292,6 +292,7 @@ void testIcebergWithKerberosAndUserImpersonation() throws IOException { Assertions.assertFalse(catalog.asSchemas().schemaExists(SCHEMA_NAME)); // Drop catalog + Assertions.assertDoesNotThrow(() -> gravitinoMetalake.disableCatalog(CATALOG_NAME)); Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME)); } diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java index 45cd2464058..ed90745a785 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java @@ -136,6 +136,7 @@ protected void startNecessaryContainers() { @AfterAll public void stop() { clearTableAndSchema(); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); client.dropMetalake(metalakeName); if (spark != null) { diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java index cf00cf5ffdb..0102611420b 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java @@ -271,7 +271,7 @@ void testPaimonWithKerberos() { Assertions.assertFalse(catalog.asSchemas().schemaExists(SCHEMA_NAME)); // Drop catalog - Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME)); + Assertions.assertTrue(gravitinoMetalake.dropCatalog(CATALOG_NAME, true)); // Drop warehouse path kerberosHiveContainer.executeInContainer( diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java index ca50af1b9ac..a2ff07e27ad 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java @@ -28,11 +28,14 @@ import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.BadRequestException; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.ConnectionFailedException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.ForbiddenException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; import org.apache.gravitino.exceptions.IllegalPrivilegeException; +import org.apache.gravitino.exceptions.InUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchFilesetException; @@ -48,6 +51,7 @@ import org.apache.gravitino.exceptions.NoSuchUserException; import org.apache.gravitino.exceptions.NonEmptySchemaException; import org.apache.gravitino.exceptions.NotFoundException; +import org.apache.gravitino.exceptions.NotInUseException; import org.apache.gravitino.exceptions.PartitionAlreadyExistsException; import org.apache.gravitino.exceptions.RESTException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; @@ -271,6 +275,14 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { + throw new CatalogNotInUseException(errorMessage); + + } else { + throw new NotInUseException(errorMessage); + } + default: super.accept(errorResponse); } @@ -311,6 +323,14 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.FORBIDDEN_CODE: throw new ForbiddenException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { + throw new CatalogNotInUseException(errorMessage); + + } else { + throw new NotInUseException(errorMessage); + } + default: super.accept(errorResponse); } @@ -351,6 +371,14 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.FORBIDDEN_CODE: throw new ForbiddenException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { + throw new CatalogNotInUseException(errorMessage); + + } else { + throw new NotInUseException(errorMessage); + } + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -394,6 +422,22 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); + case ErrorConstants.IN_USE_CODE: + if (errorResponse.getType().equals(CatalogInUseException.class.getSimpleName())) { + throw new CatalogInUseException(errorMessage); + + } else { + throw new InUseException(errorMessage); + } + + case ErrorConstants.NOT_IN_USE_CODE: + if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { + throw new CatalogNotInUseException(errorMessage); + + } else { + throw new NotInUseException(errorMessage); + } + default: super.accept(errorResponse); } @@ -512,6 +556,14 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { + throw new CatalogNotInUseException(errorMessage); + + } else { + throw new NotInUseException(errorMessage); + } + default: super.accept(errorResponse); } @@ -550,6 +602,14 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { + throw new CatalogNotInUseException(errorMessage); + + } else { + throw new NotInUseException(errorMessage); + } + default: super.accept(errorResponse); } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java index 217b8b35ab6..0f3b88133d2 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java @@ -34,6 +34,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; import org.apache.gravitino.exceptions.IllegalPrivilegeException; import org.apache.gravitino.exceptions.NoSuchCatalogException; @@ -43,6 +44,7 @@ import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.NoSuchTagException; import org.apache.gravitino.exceptions.NoSuchUserException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.exceptions.TagAlreadyExistsException; @@ -127,8 +129,19 @@ public Catalog alterCatalog(String catalogName, CatalogChange... changes) } @Override - public boolean dropCatalog(String catalogName) { - return getMetalake().dropCatalog(catalogName); + public boolean dropCatalog(String catalogName, boolean force) + throws NonEmptyEntityException, CatalogInUseException { + return getMetalake().dropCatalog(catalogName, force); + } + + @Override + public void enableCatalog(String catalogName) throws NoSuchCatalogException { + getMetalake().enableCatalog(catalogName); + } + + @Override + public void disableCatalog(String catalogName) throws NoSuchCatalogException { + getMetalake().disableCatalog(catalogName); } /** diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java index 46f75aa018b..441833bd49d 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java @@ -46,6 +46,7 @@ import org.apache.gravitino.dto.MetalakeDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.requests.CatalogCreateRequest; +import org.apache.gravitino.dto.requests.CatalogSetRequest; import org.apache.gravitino.dto.requests.CatalogUpdateRequest; import org.apache.gravitino.dto.requests.CatalogUpdatesRequest; import org.apache.gravitino.dto.requests.GroupAddRequest; @@ -77,6 +78,7 @@ import org.apache.gravitino.dto.responses.UserListResponse; import org.apache.gravitino.dto.responses.UserResponse; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; import org.apache.gravitino.exceptions.IllegalPrivilegeException; import org.apache.gravitino.exceptions.NoSuchCatalogException; @@ -86,6 +88,7 @@ import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.NoSuchTagException; import org.apache.gravitino.exceptions.NoSuchUserException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.exceptions.TagAlreadyExistsException; @@ -261,16 +264,31 @@ public Catalog alterCatalog(String catalogName, CatalogChange... changes) } /** - * Drop the catalog with specified identifier. + * Drop a catalog with specified name. If the force flag is true, it will: * - * @param catalogName the name of the catalog. - * @return true if the catalog is dropped successfully, false if the catalog does not exist. + * + * + * @param catalogName The identifier of the catalog. + * @param force Whether to force the drop. + * @return True if the catalog was dropped, false if the catalog does not exist. + * @throws NonEmptyEntityException If the catalog is not empty and force is false. + * @throws CatalogInUseException If the catalog is in use and force is false. */ @Override - public boolean dropCatalog(String catalogName) { + public boolean dropCatalog(String catalogName, boolean force) + throws NonEmptyEntityException, CatalogInUseException { + Map params = new HashMap<>(); + params.put("force", String.valueOf(force)); + DropResponse resp = restClient.delete( String.format(API_METALAKES_CATALOGS_PATH, this.name(), catalogName), + params, DropResponse.class, Collections.emptyMap(), ErrorHandlers.catalogErrorHandler()); @@ -278,6 +296,44 @@ public boolean dropCatalog(String catalogName) { return resp.dropped(); } + @Override + public void enableCatalog(String catalogName) throws NoSuchCatalogException { + CatalogSetRequest req = new CatalogSetRequest(true); + + ErrorResponse resp = + restClient.patch( + String.format(API_METALAKES_CATALOGS_PATH, this.name(), catalogName), + req, + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.catalogErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.catalogErrorHandler().accept(resp); + } + + @Override + public void disableCatalog(String catalogName) throws NoSuchCatalogException { + CatalogSetRequest req = new CatalogSetRequest(false); + + ErrorResponse resp = + restClient.patch( + String.format(API_METALAKES_CATALOGS_PATH, this.name(), catalogName), + req, + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.catalogErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.catalogErrorHandler().accept(resp); + } + /** * Test whether a catalog can be created successfully with the specified parameters, without * actually creating it. diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/HTTPClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/HTTPClient.java index 3dfc8c551f0..058d04c82d3 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/HTTPClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/HTTPClient.java @@ -561,6 +561,31 @@ public T put( Method.PUT, path, null, body, responseType, headers, errorHandler, responseHeaders); } + /** + * Sends an HTTP PATCH request to the specified path with the provided request body and processes + * the response with support for response headers. + * + * @param path The URL path to send the PATCH request to. + * @param body The REST request to place in the request body. + * @param responseType The class type of the response for deserialization (Must be registered with + * the ObjectMapper). + * @param headers A map of request headers (key-value pairs) to include in the request (can be + * null). + * @param errorHandler The error handler delegated for HTTP responses, which handles server error + * responses. + * @return The response entity parsed and converted to its type T. + * @param The class type of the response for deserialization. + */ + @Override + public T patch( + String path, + RESTRequest body, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(Method.PATCH, path, null, body, responseType, headers, errorHandler); + } + /** * Sends an HTTP DELETE request to the specified path without query parameters and processes the * response. diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/RESTClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/RESTClient.java index 191db5f8d9d..384a6358231 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/RESTClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/RESTClient.java @@ -379,6 +379,24 @@ T put( Map headers, Consumer errorHandler); + /** + * Perform a PATCH request on the specified path with given information. + * + * @param path The path to be requested. + * @param body The request body to be included in the PATCH request. + * @param responseType The class representing the type of the response. + * @param headers The headers to be included in the request. + * @param errorHandler The consumer for handling error responses. + * @return The response of the PATCH request. + * @param The type of the response. + */ + T patch( + String path, + RESTRequest body, + Class responseType, + Map headers, + Consumer errorHandler); + /** * Perform a POST request with form data on the specified path with the given information. * diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java index 78280d2a33f..728f0da4dc6 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestGravitinoMetalake.java @@ -400,7 +400,7 @@ public void testDropCatalog() throws JsonProcessingException { DropResponse resp = new DropResponse(true); buildMockResource(Method.DELETE, path, null, resp, HttpStatus.SC_OK); - boolean dropped = gravitinoClient.dropCatalog(catalogName); + boolean dropped = gravitinoClient.dropCatalog(catalogName, true); Assertions.assertTrue(dropped, "catalog should be dropped"); // Test return false diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java index 609a1b4612f..a29ef732094 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java @@ -19,6 +19,8 @@ package org.apache.gravitino.client.integration.test; +import static org.apache.gravitino.Catalog.PROPERTY_IN_USE; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.File; @@ -27,8 +29,16 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.CatalogChange; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.SchemaChange; +import org.apache.gravitino.SupportsSchemas; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; +import org.apache.gravitino.file.FilesetCatalog; +import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.BaseIT; @@ -110,7 +120,7 @@ public void testTestConnection() { Assertions.assertEquals("catalog comment", catalog.comment()); Assertions.assertTrue(catalog.properties().containsKey("metastore.uris")); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } @Test @@ -120,9 +130,9 @@ public void testDropCatalog() { Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); - Catalog catalog = - metalake.createCatalog( - catalogName, Catalog.Type.RELATIONAL, "hive", "catalog comment", properties); + metalake.createCatalog( + catalogName, Catalog.Type.RELATIONAL, "hive", "catalog comment", properties); + Catalog catalog = metalake.loadCatalog(catalogName); Assertions.assertTrue(metalake.catalogExists(catalogName)); Assertions.assertEquals(catalogName, catalog.name()); @@ -133,6 +143,79 @@ public void testDropCatalog() { catalogName, Catalog.Type.RELATIONAL, "hive", "catalog comment", properties)); Assertions.assertTrue(metalake.catalogExists(catalogName)); + Exception exception = + Assertions.assertThrows( + CatalogInUseException.class, () -> metalake.dropCatalog(catalogName)); + Assertions.assertTrue( + exception.getMessage().contains("please disable it first or use force option"), + exception.getMessage()); + + Assertions.assertDoesNotThrow(() -> metalake.disableCatalog(catalogName)); + Assertions.assertTrue(metalake.dropCatalog(catalogName), "catalog should be dropped"); + Assertions.assertFalse(metalake.dropCatalog(catalogName), "catalog should be non-existent"); + } + + @Test + public void testCatalogAvailable() { + String catalogName = GravitinoITUtils.genRandomName("test_catalog"); + Catalog catalog = + metalake.createCatalog( + catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", ImmutableMap.of()); + Assertions.assertEquals("true", catalog.properties().get(PROPERTY_IN_USE)); + + Exception exception = + Assertions.assertThrows( + CatalogInUseException.class, () -> metalake.dropCatalog(catalogName)); + Assertions.assertTrue( + exception.getMessage().contains("please disable it first or use force option"), + exception.getMessage()); + + Assertions.assertDoesNotThrow(() -> metalake.disableCatalog(catalogName)); + Catalog loadedCatalog = metalake.loadCatalog(catalogName); + Assertions.assertEquals("false", loadedCatalog.properties().get(PROPERTY_IN_USE)); + + exception = + Assertions.assertThrows( + CatalogNotInUseException.class, + () -> metalake.alterCatalog(catalogName, CatalogChange.updateComment("new comment"))); + Assertions.assertTrue( + exception.getMessage().contains("please enable it first"), exception.getMessage()); + + // test schema operations under non-in-use catalog + SupportsSchemas schemaOps = loadedCatalog.asSchemas(); + Assertions.assertThrows(CatalogNotInUseException.class, schemaOps::listSchemas); + Assertions.assertThrows( + CatalogNotInUseException.class, () -> schemaOps.createSchema("dummy", null, null)); + Assertions.assertThrows(CatalogNotInUseException.class, () -> schemaOps.loadSchema("dummy")); + Assertions.assertThrows( + CatalogNotInUseException.class, + () -> schemaOps.alterSchema("dummy", SchemaChange.removeProperty("dummy"))); + Assertions.assertThrows( + CatalogNotInUseException.class, () -> schemaOps.dropSchema("dummy", false)); + + // test fileset operations under non-in-use catalog + FilesetCatalog filesetOps = loadedCatalog.asFilesetCatalog(); + Assertions.assertThrows( + CatalogNotInUseException.class, () -> filesetOps.listFilesets(Namespace.of("dummy"))); + Assertions.assertThrows( + CatalogNotInUseException.class, + () -> filesetOps.loadFileset(NameIdentifier.of("dummy", "dummy"))); + Assertions.assertThrows( + CatalogNotInUseException.class, + () -> + filesetOps.createFileset(NameIdentifier.of("dummy", "dummy"), null, null, null, null)); + Assertions.assertThrows( + CatalogNotInUseException.class, + () -> filesetOps.dropFileset(NameIdentifier.of("dummy", "dummy"))); + Assertions.assertThrows( + CatalogNotInUseException.class, + () -> filesetOps.getFileLocation(NameIdentifier.of("dummy", "dummy"), "dummy")); + Assertions.assertThrows( + CatalogNotInUseException.class, + () -> + filesetOps.alterFileset( + NameIdentifier.of("dummy", "dummy"), FilesetChange.removeComment())); + Assertions.assertTrue(metalake.dropCatalog(catalogName), "catalog should be dropped"); Assertions.assertFalse(metalake.dropCatalog(catalogName), "catalog should be non-existent"); } @@ -151,7 +234,8 @@ public void testCreateCatalogWithoutProperties() { Assertions.assertEquals(Catalog.Type.FILESET, catalog.type()); Assertions.assertEquals("hadoop", catalog.provider()); Assertions.assertEquals("catalog comment", catalog.comment()); - Assertions.assertTrue(catalog.properties().isEmpty()); + Assertions.assertEquals("true", catalog.properties().get(PROPERTY_IN_USE)); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); // test cloud related properties @@ -182,6 +266,7 @@ public void testCreateCatalogWithoutProperties() { Assertions.assertFalse(catalog.properties().isEmpty()); Assertions.assertEquals("aws", catalog.properties().get("cloud.name")); Assertions.assertEquals("us-west-2", catalog.properties().get("cloud.region-code")); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); } @@ -201,6 +286,7 @@ public void testCreateCatalogWithChinese() { Assertions.assertEquals("这是中文comment", catalog.comment()); Assertions.assertTrue(catalog.properties().containsKey("metastore.uris")); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); } @@ -209,22 +295,18 @@ public void testListCatalogsInfo() { String relCatalogName = GravitinoITUtils.genRandomName("rel_catalog_"); Map properties = Maps.newHashMap(); properties.put("metastore.uris", hmsUri); - Catalog relCatalog = - metalake.createCatalog( - relCatalogName, - Catalog.Type.RELATIONAL, - "hive", - "relational catalog comment", - properties); + metalake.createCatalog( + relCatalogName, Catalog.Type.RELATIONAL, "hive", "relational catalog comment", properties); + Catalog relCatalog = metalake.loadCatalog(relCatalogName); String fileCatalogName = GravitinoITUtils.genRandomName("file_catalog_"); - Catalog fileCatalog = - metalake.createCatalog( - fileCatalogName, - Catalog.Type.FILESET, - "hadoop", - "file catalog comment", - Collections.emptyMap()); + metalake.createCatalog( + fileCatalogName, + Catalog.Type.FILESET, + "hadoop", + "file catalog comment", + Collections.emptyMap()); + Catalog fileCatalog = metalake.loadCatalog(fileCatalogName); Catalog[] catalogs = metalake.listCatalogsInfo(); for (Catalog catalog : catalogs) { @@ -237,7 +319,10 @@ public void testListCatalogsInfo() { Assertions.assertTrue(ArrayUtils.contains(catalogs, relCatalog)); Assertions.assertTrue(ArrayUtils.contains(catalogs, fileCatalog)); + metalake.disableCatalog(relCatalogName); metalake.dropCatalog(relCatalogName); + + metalake.disableCatalog(fileCatalogName); metalake.dropCatalog(fileCatalogName); } @@ -273,6 +358,7 @@ public void testCreateCatalogWithPackage() { Assertions.assertEquals("catalog comment", catalog.comment()); Assertions.assertTrue(catalog.properties().containsKey("package")); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); // Test using invalid package path @@ -300,12 +386,13 @@ void testUpdateCatalogWithNullableComment() { Assertions.assertTrue(metalake.catalogExists(catalogName)); Assertions.assertEquals(catalogName, catalog.name()); - Assertions.assertEquals(null, catalog.comment()); + Assertions.assertNull(catalog.comment()); Catalog updatedCatalog = metalake.alterCatalog(catalogName, CatalogChange.updateComment("new catalog comment")); Assertions.assertEquals("new catalog comment", updatedCatalog.comment()); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); } @@ -336,6 +423,7 @@ public void testAlterCatalogProperties() { Assertions.assertEquals(alterCloudName, alteredCatalog.properties().get(Catalog.CLOUD_NAME)); Assertions.assertEquals( alterRegionCode, alteredCatalog.properties().get(Catalog.CLOUD_REGION_CODE)); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); } } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java index 0cb1f9313bf..f5b5e1dab56 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java @@ -111,7 +111,7 @@ public void setUp() { public void tearDown() { relationalCatalog.asTableCatalog().dropTable(NameIdentifier.of(schema.name(), table.name())); relationalCatalog.asSchemas().dropSchema(schema.name(), true); - metalake.dropCatalog(relationalCatalog.name()); + metalake.dropCatalog(relationalCatalog.name(), true); client.dropMetalake(metalakeName); if (client != null) { diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java index 0b87eab8dbe..f5615beae06 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java @@ -146,7 +146,7 @@ public void testCreateTopic() { .createTopic(topicIdent, "comment", null, Collections.emptyMap())); catalog.asTopicCatalog().dropTopic(topicIdent); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } @Test @@ -191,7 +191,7 @@ public void testCreateFileset() { // Clean up catalog.asFilesetCatalog().dropFileset(fileIdent); catalog.asSchemas().dropSchema("schema", true); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } @Test @@ -268,6 +268,6 @@ public void testCreateTable() { // Clean up catalog.asTableCatalog().dropTable(tableIdent); catalog.asSchemas().dropSchema("schema", true); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); } } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index 877d891737f..99f1e830692 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -168,7 +168,7 @@ public void testCreateFileset() { // Clean up catalog.asFilesetCatalog().dropFileset(fileIdent); catalog.asSchemas().dropSchema("schema_owner", true); - metalake.dropCatalog(catalogNameA); + metalake.dropCatalog(catalogNameA, true); client.dropMetalake(metalakeNameA); } @@ -219,7 +219,7 @@ public void testCreateTopic() { // Clean up catalogB.asTopicCatalog().dropTopic(topicIdent); - metalake.dropCatalog(catalogNameB); + metalake.dropCatalog(catalogNameB, true); client.dropMetalake(metalakeNameB); } @@ -320,7 +320,7 @@ public void testCreateTable() { // Clean up catalog.asTableCatalog().dropTable(tableIdent); catalog.asSchemas().dropSchema("schema_owner", true); - metalake.dropCatalog(catalogNameD); + metalake.dropCatalog(catalogNameD, true); client.dropMetalake(metalakeNameD); } diff --git a/clients/client-python/gravitino/client/gravitino_client.py b/clients/client-python/gravitino/client/gravitino_client.py index 4e2b064caca..e182f5a200e 100644 --- a/clients/client-python/gravitino/client/gravitino_client.py +++ b/clients/client-python/gravitino/client/gravitino_client.py @@ -89,5 +89,11 @@ def create_catalog( def alter_catalog(self, name: str, *changes: CatalogChange): return self.get_metalake().alter_catalog(name, *changes) - def drop_catalog(self, name: str): - return self.get_metalake().drop_catalog(name) + def drop_catalog(self, name: str, force: bool = False) -> bool: + return self.get_metalake().drop_catalog(name, force) + + def enable_catalog(self, name: str): + return self.get_metalake().enable_catalog(name) + + def disable_catalog(self, name: str): + return self.get_metalake().disable_catalog(name) diff --git a/clients/client-python/gravitino/client/gravitino_metalake.py b/clients/client-python/gravitino/client/gravitino_metalake.py index e204accab84..c47412afb9e 100644 --- a/clients/client-python/gravitino/client/gravitino_metalake.py +++ b/clients/client-python/gravitino/client/gravitino_metalake.py @@ -23,6 +23,7 @@ from gravitino.dto.dto_converters import DTOConverters from gravitino.dto.metalake_dto import MetalakeDTO from gravitino.dto.requests.catalog_create_request import CatalogCreateRequest +from gravitino.dto.requests.catalog_set_request import CatalogSetRequest from gravitino.dto.requests.catalog_updates_request import CatalogUpdatesRequest from gravitino.dto.responses.catalog_list_response import CatalogListResponse from gravitino.dto.responses.catalog_response import CatalogResponse @@ -185,19 +186,59 @@ def alter_catalog(self, name: str, *changes: CatalogChange) -> Catalog: self.name(), catalog_response.catalog(), self.rest_client ) - def drop_catalog(self, name: str) -> bool: + def drop_catalog(self, name: str, force: bool = False) -> bool: """Drop the catalog with specified name. Args: - name the name of the catalog. + name: the name of the catalog. + force: whether to force drop the catalog. Returns: - true if the catalog is dropped successfully, false otherwise. + true if the catalog is dropped successfully, false if the catalog does not exist. """ + params = {"force": str(force)} url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) - response = self.rest_client.delete(url, error_handler=CATALOG_ERROR_HANDLER) + response = self.rest_client.delete( + url, params=params, error_handler=CATALOG_ERROR_HANDLER + ) drop_response = DropResponse.from_json(response.body, infer_missing=True) drop_response.validate() return drop_response.dropped() + + def enable_catalog(self, name: str): + """Enable the catalog with specified name. If the catalog is already in use, this method does nothing. + + Args: + name: the name of the catalog. + + Raises: + NoSuchCatalogException if the catalog with specified name does not exist. + """ + + catalog_enable_request = CatalogSetRequest(in_use=True) + catalog_enable_request.validate() + + url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) + self.rest_client.patch( + url, json=catalog_enable_request, error_handler=CATALOG_ERROR_HANDLER + ) + + def disable_catalog(self, name: str): + """Disable the catalog with specified name. If the catalog is already disabled, this method does nothing. + + Args: + name: the name of the catalog. + + Raises: + NoSuchCatalogException if the catalog with specified name does not exist. + """ + + catalog_disable_request = CatalogSetRequest(in_use=False) + catalog_disable_request.validate() + + url = self.API_METALAKES_CATALOGS_PATH.format(self.name(), name) + self.rest_client.patch( + url, json=catalog_disable_request, error_handler=CATALOG_ERROR_HANDLER + ) diff --git a/clients/client-python/gravitino/constants/error.py b/clients/client-python/gravitino/constants/error.py index 574aba73bbf..7ff56c6fee8 100644 --- a/clients/client-python/gravitino/constants/error.py +++ b/clients/client-python/gravitino/constants/error.py @@ -56,6 +56,12 @@ class ErrorConstants(IntEnum): # Error codes for connect to catalog failed. CONNECTION_FAILED_CODE = 1007 + # Error codes for operation on a no in use entity. + NOT_IN_USE_CODE = 1009 + + # Error codes for drop an in use entity. + IN_USE_CODE = 1010 + # Error codes for invalid state. UNKNOWN_ERROR_CODE = 1100 diff --git a/clients/client-python/gravitino/dto/requests/catalog_set_request.py b/clients/client-python/gravitino/dto/requests/catalog_set_request.py new file mode 100644 index 00000000000..b93d8bd0f30 --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/catalog_set_request.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field + +from dataclasses_json import config + +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class CatalogSetRequest(RESTRequest): + """Represents a request to set a catalog in use.""" + + _in_use: bool = field(metadata=config(field_name="inUse")) + + def __init__(self, in_use: bool): + self._in_use = in_use + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if in_use is not set. + """ + if self._in_use is None: + raise ValueError('"in_use" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/exceptions/base.py b/clients/client-python/gravitino/exceptions/base.py index 45bcf23d04c..2dff76ac4ad 100644 --- a/clients/client-python/gravitino/exceptions/base.py +++ b/clients/client-python/gravitino/exceptions/base.py @@ -93,6 +93,22 @@ class NonEmptySchemaException(NotEmptyException): """Exception thrown when a namespace is not empty.""" +class InUseException(GravitinoRuntimeException): + """Base class for all exceptions thrown when an entity is in use and cannot be deleted.""" + + +class CatalogInUseException(InUseException): + """An Exception thrown when a catalog is in use and cannot be deleted.""" + + +class NotInUseException(GravitinoRuntimeException): + """Base class for all exceptions thrown when an entity is not in use.""" + + +class CatalogNotInUseException(NotInUseException): + """An exception thrown when operating on not in use catalog.""" + + class UnsupportedOperationException(GravitinoRuntimeException): """Base class for all exceptions thrown when an operation is unsupported""" diff --git a/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py b/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py index 1523185ef0b..7835f967736 100644 --- a/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py +++ b/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py @@ -20,9 +20,11 @@ from gravitino.exceptions.handlers.rest_error_handler import RestErrorHandler from gravitino.exceptions.base import ( ConnectionFailedException, + CatalogInUseException, NoSuchMetalakeException, NoSuchCatalogException, CatalogAlreadyExistsException, + CatalogNotInUseException, ) @@ -36,14 +38,24 @@ def handle(self, error_response: ErrorResponse): if code == ErrorConstants.CONNECTION_FAILED_CODE: raise ConnectionFailedException(error_message) + if code == ErrorConstants.NOT_FOUND_CODE: if exception_type == NoSuchMetalakeException.__name__: raise NoSuchMetalakeException(error_message) if exception_type == NoSuchCatalogException.__name__: raise NoSuchCatalogException(error_message) + if code == ErrorConstants.ALREADY_EXISTS_CODE: raise CatalogAlreadyExistsException(error_message) + if code == ErrorConstants.IN_USE_CODE: + if exception_type == CatalogInUseException.__name__: + raise CatalogInUseException(error_message) + + if code == ErrorConstants.NOT_IN_USE_CODE: + if exception_type == CatalogNotInUseException.__name__: + raise CatalogNotInUseException(error_message) + super().handle(error_response) diff --git a/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py b/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py index bf76114f33c..95baa798cef 100644 --- a/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py +++ b/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py @@ -18,7 +18,11 @@ from gravitino.constants.error import ErrorConstants from gravitino.dto.responses.error_response import ErrorResponse from gravitino.exceptions.handlers.rest_error_handler import RestErrorHandler -from gravitino.exceptions.base import NoSuchFilesetException, NoSuchSchemaException +from gravitino.exceptions.base import ( + NoSuchFilesetException, + NoSuchSchemaException, + CatalogNotInUseException, +) class FilesetErrorHandler(RestErrorHandler): @@ -35,6 +39,9 @@ def handle(self, error_response: ErrorResponse): if exception_type == NoSuchFilesetException.__name__: raise NoSuchFilesetException(error_message) + if code == ErrorConstants.NOT_IN_USE_CODE: + raise CatalogNotInUseException(error_message) + super().handle(error_response) diff --git a/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py b/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py index 0a55cc52940..fa805de81dc 100644 --- a/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py +++ b/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py @@ -21,6 +21,7 @@ from gravitino.exceptions.base import ( NoSuchCatalogException, NoSuchSchemaException, + CatalogNotInUseException, SchemaAlreadyExistsException, NonEmptySchemaException, ) @@ -39,12 +40,16 @@ def handle(self, error_response: ErrorResponse): raise NoSuchCatalogException(error_message) if exception_type == NoSuchSchemaException.__name__: raise NoSuchSchemaException(error_message) + if code == ErrorConstants.ALREADY_EXISTS_CODE: raise SchemaAlreadyExistsException(error_message) if code == ErrorConstants.NON_EMPTY_CODE: raise NonEmptySchemaException(error_message) + if code == ErrorConstants.NOT_IN_USE_CODE: + raise CatalogNotInUseException(error_message) + super().handle(error_response) diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py index 262c73c2b4c..b3e38a5d359 100644 --- a/clients/client-python/gravitino/utils/http_client.py +++ b/clients/client-python/gravitino/utils/http_client.py @@ -240,6 +240,11 @@ def put(self, endpoint, json=None, error_handler=None, **kwargs): "put", endpoint, json=json, error_handler=error_handler, **kwargs ) + def patch(self, endpoint, json=None, error_handler=None, **kwargs): + return self._request( + "patch", endpoint, json=json, error_handler=error_handler, **kwargs + ) + def post_form(self, endpoint, data=None, error_handler=None, **kwargs): return self._request( "post", endpoint, data=data, error_handler=error_handler, **kwargs diff --git a/clients/client-python/tests/integration/auth/test_auth_common.py b/clients/client-python/tests/integration/auth/test_auth_common.py index a8cdc7c1e2d..2f1506319a7 100644 --- a/clients/client-python/tests/integration/auth/test_auth_common.py +++ b/clients/client-python/tests/integration/auth/test_auth_common.py @@ -92,7 +92,7 @@ def clean_test_data(self): logger.info( "Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(name=self.catalog_name), + self.gravitino_client.drop_catalog(name=self.catalog_name, force=True), ) except GravitinoRuntimeException: logger.warning("Failed to drop catalog %s", self.catalog_name) diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 71caafbc206..58580b8c518 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -43,6 +43,7 @@ class TestCatalog(IntegrationTestEnv): catalog_comment: str = "catalogComment" catalog_location_prop: str = "location" # Fileset Catalog must set `location` catalog_provider: str = "hadoop" + catalog_in_use_prop: str = "in-use" catalog_ident: NameIdentifier = NameIdentifier.of(metalake_name, catalog_name) @@ -82,7 +83,7 @@ def clean_test_data(self): logger.info( "Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(name=self.catalog_name), + self.gravitino_client.drop_catalog(name=self.catalog_name, force=True), ) except GravitinoRuntimeException: logger.warning("Failed to drop catalog %s", self.catalog_name) @@ -105,7 +106,11 @@ def test_create_catalog(self): catalog = self.create_catalog(self.catalog_name) self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual( - catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} + catalog.properties(), + { + self.catalog_location_prop: "/tmp/test_schema", + self.catalog_in_use_prop: "true", + }, ) def test_failed_create_catalog(self): @@ -140,6 +145,7 @@ def test_alter_catalog(self): def test_drop_catalog(self): self.create_catalog(self.catalog_name) + self.gravitino_client.disable_catalog(self.catalog_name) self.assertTrue(self.gravitino_client.drop_catalog(name=self.catalog_name)) def test_list_catalogs_info(self): @@ -154,7 +160,11 @@ def test_load_catalog(self): self.assertEqual(catalog.name(), self.catalog_name) self.assertEqual(catalog.comment(), self.catalog_comment) self.assertEqual( - catalog.properties(), {self.catalog_location_prop: "/tmp/test_schema"} + catalog.properties(), + { + self.catalog_location_prop: "/tmp/test_schema", + self.catalog_in_use_prop: "true", + }, ) self.assertEqual(catalog.audit_info().creator(), "anonymous") diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 0e92ec1b090..2696c170a0d 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -119,7 +119,7 @@ def clean_test_data(self): logger.info( "Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(name=self.catalog_name), + self.gravitino_client.drop_catalog(name=self.catalog_name, force=True), ) except GravitinoRuntimeException: logger.warning("Failed to drop catalog %s", self.catalog_name) diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 9116005b840..8bc6597b455 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -193,7 +193,7 @@ def _clean_test_data(cls): logger.info( "Drop catalog %s[%s]", cls.catalog_name, - cls.gravitino_client.drop_catalog(name=cls.catalog_name), + cls.gravitino_client.drop_catalog(name=cls.catalog_name, force=True), ) except GravitinoRuntimeException: logger.warning("Failed to drop catalog %s", cls.catalog_name) diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index e57e6676b00..269693dcf25 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -119,7 +119,7 @@ def clean_test_data(self): logger.info( "Drop catalog %s[%s]", self.catalog_ident, - self.gravitino_client.drop_catalog(name=self.catalog_name), + self.gravitino_client.drop_catalog(name=self.catalog_name, force=True), ) except GravitinoRuntimeException: logger.warning("Failed to drop catalog %s", self.catalog_name) diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java index ced9a0b8b89..ef41637d6f9 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java @@ -91,7 +91,7 @@ public void startUp() throws Exception { public void tearDown() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); client.dropMetalake(metalakeName); if (client != null) { diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogSetRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogSetRequest.java new file mode 100644 index 00000000000..d2c9c744b2f --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogSetRequest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to set a catalog in use. */ +@Getter +@EqualsAndHashCode +@ToString +public class CatalogSetRequest implements RESTRequest { + + @JsonProperty("inUse") + private final boolean inUse; + + /** Default constructor for CatalogSetRequest. */ + public CatalogSetRequest() { + this(false); + } + + /** + * Constructor for CatalogSetRequest. + * + * @param inUse The in use status to set. + */ + public CatalogSetRequest(boolean inUse) { + this.inUse = inUse; + } + + /** + * Validates the request. No validation needed. + * + * @throws IllegalArgumentException If the request is invalid. + */ + @Override + public void validate() throws IllegalArgumentException { + // No validation needed + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ErrorConstants.java b/common/src/main/java/org/apache/gravitino/dto/responses/ErrorConstants.java index db799ac187c..4953463d74b 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/ErrorConstants.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ErrorConstants.java @@ -48,6 +48,12 @@ public class ErrorConstants { /** Error codes for forbidden operation. */ public static final int FORBIDDEN_CODE = 1008; + /** Error codes for operation on a no in use entity. */ + public static final int NOT_IN_USE_CODE = 1009; + + /** Error codes for drop an in use entity. */ + public static final int IN_USE_CODE = 1010; + /** Error codes for invalid state. */ public static final int UNKNOWN_ERROR_CODE = 1100; diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ErrorResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ErrorResponse.java index 2c1e1e9ef84..c68b71f1357 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/ErrorResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ErrorResponse.java @@ -234,6 +234,31 @@ public static ErrorResponse alreadyExists(String type, String message, Throwable ErrorConstants.ALREADY_EXISTS_CODE, type, message, getStackTrace(throwable)); } + /** + * Create a new entity in use error instance of {@link ErrorResponse}. + * + * @param type The type of the error. + * @param message The message of the error. + * @param throwable The throwable that caused the error. + * @return The new instance. + */ + public static ErrorResponse notInUse(String type, String message, Throwable throwable) { + return new ErrorResponse( + ErrorConstants.NOT_IN_USE_CODE, type, message, getStackTrace(throwable)); + } + + /** + * Create a new entity in use error instance of {@link ErrorResponse}. + * + * @param type The type of the error. + * @param message The message of the error. + * @param throwable The throwable that caused the error. + * @return The new instance. + */ + public static ErrorResponse inUse(String type, String message, Throwable throwable) { + return new ErrorResponse(ErrorConstants.IN_USE_CODE, type, message, getStackTrace(throwable)); + } + /** * Create a new non-empty error instance of {@link ErrorResponse}. * diff --git a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java index a27df871040..77d76b47476 100644 --- a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java +++ b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java @@ -366,14 +366,18 @@ private void initGravitinoServerComponents() { // create and initialize a random id generator this.idGenerator = new RandomIdGenerator(); - // Create and initialize metalake related modules + // Create and initialize metalake related modules, the operation chain is: + // MetalakeEventDispatcher -> MetalakeNormalizeDispatcher -> MetalakeHookDispatcher -> + // MetalakeManager MetalakeDispatcher metalakeManager = new MetalakeManager(entityStore, idGenerator); MetalakeHookDispatcher metalakeHookDispatcher = new MetalakeHookDispatcher(metalakeManager); MetalakeNormalizeDispatcher metalakeNormalizeDispatcher = new MetalakeNormalizeDispatcher(metalakeHookDispatcher); this.metalakeDispatcher = new MetalakeEventDispatcher(eventBus, metalakeNormalizeDispatcher); - // Create and initialize Catalog related modules + // Create and initialize Catalog related modules, the operation chain is: + // CatalogEventDispatcher -> CatalogNormalizeDispatcher -> CatalogHookDispatcher -> + // CatalogManager this.catalogManager = new CatalogManager(config, entityStore, idGenerator); CatalogHookDispatcher catalogHookDispatcher = new CatalogHookDispatcher(catalogManager); CatalogNormalizeDispatcher catalogNormalizeDispatcher = diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index 9dc1d0ed277..f03b500bb2d 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -18,10 +18,11 @@ */ package org.apache.gravitino.catalog; +import static org.apache.gravitino.Catalog.PROPERTY_IN_USE; import static org.apache.gravitino.StringIdentifier.DUMMY_ID; -import static org.apache.gravitino.StringIdentifier.ID_KEY; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import static org.apache.gravitino.connector.BaseCatalogPropertiesMetadata.BASIC_CATALOG_PROPERTIES_METADATA; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -31,7 +32,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Multimaps; import com.google.common.collect.Streams; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.Closeable; @@ -45,6 +45,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.ServiceLoader; @@ -52,6 +53,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -76,10 +78,13 @@ import org.apache.gravitino.connector.SupportsSchemas; import org.apache.gravitino.connector.capability.Capability; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.GravitinoRuntimeException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.messaging.TopicCatalog; import org.apache.gravitino.meta.AuditInfo; @@ -103,6 +108,12 @@ public class CatalogManager implements CatalogDispatcher, Closeable { private static final Logger LOG = LoggerFactory.getLogger(CatalogManager.class); + public static boolean catalogInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException, NoSuchCatalogException { + // todo: check if the metalake is in use + return getInUseValue(store, ident); + } + /** Wrapper class for a catalog instance and its class loader. */ public static class CatalogWrapper { private BaseCatalog catalog; @@ -295,15 +306,10 @@ public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeExce List catalogEntities = store.list(namespace, CatalogEntity.class, EntityType.CATALOG); - // Using provider as key to avoid loading the same type catalog instance multiple times - Map> hiddenProps = new HashMap<>(); - Multimaps.index(catalogEntities, CatalogEntity::getProvider) - .asMap() - .forEach((p, e) -> hiddenProps.put(p, getHiddenPropertyNames(e.iterator().next()))); - return catalogEntities.stream() - .map(e -> e.toCatalogInfoWithoutHiddenProps(hiddenProps.get(e.getProvider()))) + .map(e -> e.toCatalogInfoWithResolvedProps(getResolvedProperties(e))) .toArray(Catalog[]::new); + } catch (IOException ioe) { LOG.error("Failed to list catalogs in metalake {}", metalakeIdent, ioe); throw new RuntimeException(ioe); @@ -375,7 +381,7 @@ public Catalog createCatalog( } store.put(e, false /* overwrite */); - CatalogWrapper wrapper = catalogCache.get(ident, id -> createCatalogWrapper(e)); + CatalogWrapper wrapper = catalogCache.get(ident, id -> createCatalogWrapper(e, mergedConfig)); needClean = false; return wrapper.catalog; @@ -457,7 +463,7 @@ public void testConnection( .build()) .build(); - CatalogWrapper wrapper = createCatalogWrapper(dummyEntity); + CatalogWrapper wrapper = createCatalogWrapper(dummyEntity, mergedConfig); wrapper.doWithCatalogOps( c -> { c.testConnection(ident, type, provider, comment, mergedConfig); @@ -474,6 +480,64 @@ public void testConnection( } } + @Override + public void enableCatalog(NameIdentifier ident) + throws NoSuchCatalogException, CatalogNotInUseException { + try { + if (catalogInUse(store, ident)) { + return; + } + + store.update( + ident, + CatalogEntity.class, + EntityType.CATALOG, + catalog -> { + CatalogEntity.Builder newCatalogBuilder = newCatalogBuilder(ident.namespace(), catalog); + + Map newProps = + catalog.getProperties() == null + ? new HashMap<>() + : new HashMap<>(catalog.getProperties()); + newProps.put(PROPERTY_IN_USE, "true"); + newCatalogBuilder.withProperties(newProps); + + return newCatalogBuilder.build(); + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { + try { + if (!catalogInUse(store, ident)) { + return; + } + store.update( + ident, + CatalogEntity.class, + EntityType.CATALOG, + catalog -> { + CatalogEntity.Builder newCatalogBuilder = newCatalogBuilder(ident.namespace(), catalog); + + Map newProps = + catalog.getProperties() == null + ? new HashMap<>() + : new HashMap<>(catalog.getProperties()); + newProps.put(PROPERTY_IN_USE, "false"); + newCatalogBuilder.withProperties(newProps); + + return newCatalogBuilder.build(); + }); + catalogCache.invalidate(ident); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** * Alters an existing catalog with the specified changes. * @@ -486,6 +550,10 @@ public void testConnection( @Override public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) throws NoSuchCatalogException, IllegalArgumentException { + if (!catalogInUse(store, ident)) { + throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); + } + // There could be a race issue that someone is using the catalog from cache while we are // updating it. @@ -519,22 +587,7 @@ public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) EntityType.CATALOG, catalog -> { CatalogEntity.Builder newCatalogBuilder = - CatalogEntity.builder() - .withId(catalog.id()) - .withName(catalog.name()) - .withNamespace(ident.namespace()) - .withType(catalog.getType()) - .withProvider(catalog.getProvider()) - .withComment(catalog.getComment()); - - AuditInfo newInfo = - AuditInfo.builder() - .withCreator(catalog.auditInfo().creator()) - .withCreateTime(catalog.auditInfo().createTime()) - .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) - .withLastModifiedTime(Instant.now()) - .build(); - newCatalogBuilder.withAuditInfo(newInfo); + newCatalogBuilder(ident.namespace(), catalog); Map newProps = catalog.getProperties() == null @@ -544,8 +597,10 @@ public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) return newCatalogBuilder.build(); }); - return catalogCache.get( - updatedCatalog.nameIdentifier(), id -> createCatalogWrapper(updatedCatalog)) + return Objects.requireNonNull( + catalogCache.get( + updatedCatalog.nameIdentifier(), + id -> createCatalogWrapper(updatedCatalog, null))) .catalog; } catch (NoSuchEntityException ne) { @@ -562,38 +617,39 @@ public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) } } - /** - * Drops (deletes) the catalog with the specified identifier. - * - * @param ident The identifier of the catalog to drop. - * @return {@code true} if the catalog was successfully dropped, {@code false} otherwise. - */ @Override - public boolean dropCatalog(NameIdentifier ident) { - // There could be a race issue that someone is using the catalog while we are dropping it. - catalogCache.invalidate(ident); - + public boolean dropCatalog(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, CatalogInUseException { try { + boolean catalogInUse = catalogInUse(store, ident); + if (catalogInUse && !force) { + throw new CatalogInUseException( + "Catalog %s is in use, please disable it first or use force option", ident); + } + + List schemas = + store.list( + Namespace.of(ident.namespace().level(0), ident.name()), + SchemaEntity.class, + EntityType.SCHEMA); CatalogEntity catalogEntity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); - if (catalogEntity.getProvider().equals("kafka")) { - // Kafka catalog needs to cascade drop the default schema - List schemas = - store.list( - Namespace.of(ident.namespace().level(0), ident.name()), - SchemaEntity.class, - EntityType.SCHEMA); - // If there is only one schema, it must be the default schema, because we don't allow to - // drop the default schema. - if (schemas.size() == 1) { - return store.delete(ident, EntityType.CATALOG, true); + + if (!schemas.isEmpty() && !force) { + // the Kafka catalog is special, it includes a default schema + if (!catalogEntity.getProvider().equals("kafka") || schemas.size() > 1) { + throw new NonEmptyEntityException( + "Catalog %s has schemas, please drop them first or use force option", ident); } } - return store.delete(ident, EntityType.CATALOG); - } catch (NoSuchEntityException e) { + + catalogCache.invalidate(ident); + return store.delete(ident, EntityType.CATALOG, true); + + } catch (NoSuchMetalakeException | NoSuchCatalogException ignored) { return false; - } catch (IOException ioe) { - LOG.error("Failed to drop catalog {}", ident, ioe); - throw new RuntimeException(ioe); + + } catch (IOException e) { + throw new RuntimeException(e); } } @@ -609,6 +665,44 @@ public CatalogWrapper loadCatalogAndWrap(NameIdentifier ident) throws NoSuchCata return catalogCache.get(ident, this::loadCatalogInternal); } + private static boolean getInUseValue(EntityStore store, NameIdentifier catalogIdent) { + try { + CatalogEntity catalogEntity = + store.get(catalogIdent, EntityType.CATALOG, CatalogEntity.class); + return (boolean) + BASIC_CATALOG_PROPERTIES_METADATA.getOrDefault( + catalogEntity.getProperties(), PROPERTY_IN_USE); + + } catch (NoSuchEntityException e) { + LOG.warn("Catalog {} does not exist", catalogIdent, e); + throw new NoSuchCatalogException(CATALOG_DOES_NOT_EXIST_MSG, catalogIdent); + + } catch (IOException e) { + LOG.error("Failed to do store operation", e); + throw new RuntimeException(e); + } + } + + private CatalogEntity.Builder newCatalogBuilder(Namespace namespace, CatalogEntity catalog) { + CatalogEntity.Builder builder = + CatalogEntity.builder() + .withId(catalog.id()) + .withName(catalog.name()) + .withNamespace(namespace) + .withType(catalog.getType()) + .withProvider(catalog.getProvider()) + .withComment(catalog.getComment()); + + AuditInfo newInfo = + AuditInfo.builder() + .withCreator(catalog.auditInfo().creator()) + .withCreateTime(catalog.auditInfo().createTime()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) + .withLastModifiedTime(Instant.now()) + .build(); + return builder.withAuditInfo(newInfo); + } + private Map buildCatalogConf(String provider, Map properties) { Map newProperties = Optional.ofNullable(properties).orElse(Maps.newHashMap()); // load catalog-related configuration from catalog-specific configuration file @@ -650,7 +744,7 @@ private void checkMetalakeExists(NameIdentifier ident) throws NoSuchMetalakeExce private CatalogWrapper loadCatalogInternal(NameIdentifier ident) throws NoSuchCatalogException { try { CatalogEntity entity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); - return createCatalogWrapper(entity); + return createCatalogWrapper(entity, null); } catch (NoSuchEntityException ne) { LOG.warn("Catalog {} does not exist", ident, ne); @@ -662,7 +756,16 @@ private CatalogWrapper loadCatalogInternal(NameIdentifier ident) throws NoSuchCa } } - private CatalogWrapper createCatalogWrapper(CatalogEntity entity) { + /** + * Create a catalog wrapper from the catalog entity and validate the given properties for + * creation. The properties can be null if it is not needed to validate. + * + * @param entity The catalog entity. + * @param propsToValidate The properties to validate. + * @return The created catalog wrapper. + */ + private CatalogWrapper createCatalogWrapper( + CatalogEntity entity, @Nullable Map propsToValidate) { Map conf = entity.getProperties(); String provider = entity.getProvider(); @@ -673,9 +776,7 @@ private CatalogWrapper createCatalogWrapper(CatalogEntity entity) { // Validate catalog properties and initialize the config classLoader.withClassLoader( cl -> { - Map configWithoutId = Maps.newHashMap(conf); - configWithoutId.remove(ID_KEY); - validatePropertyForCreate(catalog.catalogPropertiesMetadata(), configWithoutId); + validatePropertyForCreate(catalog.catalogPropertiesMetadata(), propsToValidate); // Call wrapper.catalog.properties() to make BaseCatalog#properties in IsolatedClassLoader // not null. Why do we do this? Because wrapper.catalog.properties() needs to be called in @@ -691,6 +792,23 @@ private CatalogWrapper createCatalogWrapper(CatalogEntity entity) { return wrapper; } + /** + * Get the resolved properties (filter out the hidden properties and add some required default + * properties) of the catalog entity. + * + * @param entity The catalog entity. + * @return The resolved properties. + */ + private Map getResolvedProperties(CatalogEntity entity) { + Map conf = entity.getProperties(); + String provider = entity.getProvider(); + + try (IsolatedClassLoader classLoader = createClassLoader(provider, conf)) { + BaseCatalog catalog = createBaseCatalog(classLoader, entity); + return classLoader.withClassLoader(cl -> catalog.properties(), RuntimeException.class); + } + } + private Set getHiddenPropertyNames(CatalogEntity entity) { Map conf = entity.getProperties(); String provider = entity.getProvider(); diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogNormalizeDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogNormalizeDispatcher.java index 32923acb8ec..01bfc8f9e23 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogNormalizeDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogNormalizeDispatcher.java @@ -30,8 +30,10 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; public class CatalogNormalizeDispatcher implements CatalogDispatcher { private static final Set RESERVED_WORDS = @@ -104,6 +106,12 @@ public boolean dropCatalog(NameIdentifier ident) { return dispatcher.dropCatalog(ident); } + @Override + public boolean dropCatalog(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, CatalogInUseException { + return dispatcher.dropCatalog(ident, force); + } + @Override public void testConnection( NameIdentifier ident, @@ -116,6 +124,16 @@ public void testConnection( dispatcher.testConnection(ident, type, provider, comment, properties); } + @Override + public void enableCatalog(NameIdentifier ident) throws NoSuchCatalogException { + dispatcher.enableCatalog(ident); + } + + @Override + public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { + dispatcher.disableCatalog(ident); + } + private void validateCatalogName(String name) throws IllegalArgumentException { if (RESERVED_WORDS.contains(name.toLowerCase())) { throw new IllegalArgumentException("The catalog name '" + name + "' is reserved."); diff --git a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java index a9f875df68c..88add624870 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.catalog; +import static org.apache.gravitino.catalog.CatalogManager.catalogInUse; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; import static org.apache.gravitino.utils.NameIdentifierUtil.getCatalogIdentifier; @@ -34,6 +35,7 @@ import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.PropertiesMetadata; import org.apache.gravitino.connector.capability.Capability; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.messaging.TopicChange; @@ -92,6 +94,10 @@ protected R doWithTable( protected R doWithCatalog( NameIdentifier ident, ThrowableFunction fn, Class ex) throws E { + if (!catalogInUse(store, ident)) { + throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); + } + try { CatalogManager.CatalogWrapper c = catalogManager.loadCatalogAndWrap(ident); return fn.apply(c); @@ -112,6 +118,10 @@ protected R doWithCatalog( Class ex1, Class ex2) throws E1, E2 { + if (!catalogInUse(store, ident)) { + throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); + } + try { CatalogManager.CatalogWrapper c = catalogManager.loadCatalogAndWrap(ident); return fn.apply(c); diff --git a/core/src/main/java/org/apache/gravitino/catalog/SupportsCatalogs.java b/core/src/main/java/org/apache/gravitino/catalog/SupportsCatalogs.java index 33f5d40946c..c83c70cb822 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/SupportsCatalogs.java +++ b/core/src/main/java/org/apache/gravitino/catalog/SupportsCatalogs.java @@ -26,8 +26,11 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Interface for supporting catalogs. It includes methods for listing, loading, creating, altering @@ -115,12 +118,45 @@ Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) throws NoSuchCatalogException, IllegalArgumentException; /** - * Drop a catalog with specified identifier. + * Drop a catalog with specified identifier. Please make sure: + * + *
    + *
  • There is no schema in the catalog. Otherwise, a {@link NonEmptyEntityException} will be + * thrown. + *
  • The method {@link #disableCatalog(NameIdentifier)} has been called before dropping the + * catalog. + *
+ * + * It is equivalent to calling {@code dropCatalog(ident, false)}. * * @param ident the identifier of the catalog. * @return True if the catalog was dropped, false if the catalog does not exist. + * @throws NonEmptyEntityException If the catalog is not empty. + * @throws CatalogInUseException If the catalog is in use. + */ + default boolean dropCatalog(NameIdentifier ident) + throws NonEmptyEntityException, CatalogInUseException { + return dropCatalog(ident, false); + } + + /** + * Drop a catalog with specified identifier. If the force flag is true, it will: + * + *
    + *
  • Cascade drop all sub-entities (schemas, tables, etc.) of the catalog in Gravitino store. + *
  • Drop the catalog even if it is in use. + *
  • External resources (e.g. database, table, etc.) associated with sub-entities will not be + * dropped unless it is managed (such as managed fileset). + *
+ * + * @param ident The identifier of the catalog. + * @param force Whether to force the drop. + * @return True if the catalog was dropped, false if the catalog does not exist. + * @throws NonEmptyEntityException If the catalog is not empty and force is false. + * @throws CatalogInUseException If the catalog is in use and force is false. */ - boolean dropCatalog(NameIdentifier ident); + boolean dropCatalog(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, CatalogInUseException; /** * Test whether the catalog with specified parameters can be connected to before creating it. @@ -139,4 +175,29 @@ void testConnection( String comment, Map properties) throws Exception; + + /** + * Enable a catalog. If the catalog is already enabled, this method does nothing. + * + * @param ident The identifier of the catalog. + * @throws NoSuchCatalogException If the catalog does not exist. + * @throws CatalogNotInUseException If its parent metalake is not in use. + */ + void enableCatalog(NameIdentifier ident) throws NoSuchCatalogException, CatalogNotInUseException; + + /** + * Disable a catalog. If the catalog is already disabled, this method does nothing. Once a catalog + * is disabled: + * + *
    + *
  • It can only be listed, loaded, dropped, or disable. + *
  • Any other operations on the catalog will throw an {@link CatalogNotInUseException}. + *
  • Any operation on the sub-entities (schemas, tables, etc.) will throw an {@link + * CatalogNotInUseException}. + *
+ * + * @param ident The identifier of the catalog. + * @throws NoSuchCatalogException If the catalog does not exist. + */ + void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException; } diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index bb3c2f9bd7f..213afd4fafc 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -342,6 +342,9 @@ public Map properties() { tempProperties .entrySet() .removeIf(entry -> catalogPropertiesMetadata().isHiddenProperty(entry.getKey())); + tempProperties.putIfAbsent( + PROPERTY_IN_USE, + catalogPropertiesMetadata().getDefaultValue(PROPERTY_IN_USE).toString()); properties = tempProperties; } } diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalogPropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalogPropertiesMetadata.java index 1ba3e560640..28a95432f91 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalogPropertiesMetadata.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalogPropertiesMetadata.java @@ -21,12 +21,14 @@ import static org.apache.gravitino.Catalog.CLOUD_NAME; import static org.apache.gravitino.Catalog.CLOUD_REGION_CODE; +import static org.apache.gravitino.Catalog.PROPERTY_IN_USE; import static org.apache.gravitino.Catalog.PROPERTY_PACKAGE; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import java.util.Collections; import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.annotation.Evolving; @@ -34,6 +36,14 @@ @Evolving public abstract class BaseCatalogPropertiesMetadata extends BasePropertiesMetadata { + public static final PropertiesMetadata BASIC_CATALOG_PROPERTIES_METADATA = + new BaseCatalogPropertiesMetadata() { + @Override + protected Map> specificPropertyEntries() { + return Collections.emptyMap(); + } + }; + // The basic property entries for catalog entities private static final Map> BASIC_CATALOG_PROPERTY_ENTRIES = Maps.uniqueIndex( @@ -73,6 +83,11 @@ public abstract class BaseCatalogPropertiesMetadata extends BasePropertiesMetada "The region code of the cloud that the catalog is running on", false /* immutable */, null /* The default value does not work because if the user does not set it, this property will not be displayed */, + false /* hidden */), + PropertyEntry.booleanReservedPropertyEntry( + PROPERTY_IN_USE, + "The property indicating the catalog is in use", + true /* default value */, false /* hidden */)), PropertyEntry::getName); diff --git a/core/src/main/java/org/apache/gravitino/hook/CatalogHookDispatcher.java b/core/src/main/java/org/apache/gravitino/hook/CatalogHookDispatcher.java index 7a9989f4edf..3dc2bc2bdab 100644 --- a/core/src/main/java/org/apache/gravitino/hook/CatalogHookDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/hook/CatalogHookDispatcher.java @@ -32,8 +32,11 @@ import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.connector.BaseCatalog; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.PrincipalUtils; @@ -109,6 +112,12 @@ public boolean dropCatalog(NameIdentifier ident) { return dispatcher.dropCatalog(ident); } + @Override + public boolean dropCatalog(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, CatalogInUseException { + return dispatcher.dropCatalog(ident, force); + } + @Override public void testConnection( NameIdentifier ident, @@ -120,6 +129,17 @@ public void testConnection( dispatcher.testConnection(ident, type, provider, comment, properties); } + @Override + public void enableCatalog(NameIdentifier ident) + throws NoSuchCatalogException, CatalogNotInUseException { + dispatcher.enableCatalog(ident); + } + + @Override + public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { + dispatcher.disableCatalog(ident); + } + @Override public boolean catalogExists(NameIdentifier ident) { return dispatcher.catalogExists(ident); diff --git a/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java index d6cc786f5bd..04a2600d8ec 100644 --- a/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java @@ -26,8 +26,11 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.listener.api.event.AlterCatalogEvent; import org.apache.gravitino.listener.api.event.AlterCatalogFailureEvent; import org.apache.gravitino.listener.api.event.CreateCatalogEvent; @@ -145,9 +148,10 @@ public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) } @Override - public boolean dropCatalog(NameIdentifier ident) { + public boolean dropCatalog(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, CatalogInUseException { try { - boolean isExists = dispatcher.dropCatalog(ident); + boolean isExists = dispatcher.dropCatalog(ident, force); eventBus.dispatchEvent( new DropCatalogEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); return isExists; @@ -169,4 +173,17 @@ public void testConnection( // TODO: Support event dispatching for testConnection dispatcher.testConnection(ident, type, provider, comment, properties); } + + @Override + public void enableCatalog(NameIdentifier ident) + throws NoSuchCatalogException, CatalogNotInUseException { + // todo: support activate catalog event + dispatcher.enableCatalog(ident); + } + + @Override + public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { + // todo: support disable catalog event + dispatcher.disableCatalog(ident); + } } diff --git a/core/src/main/java/org/apache/gravitino/meta/CatalogEntity.java b/core/src/main/java/org/apache/gravitino/meta/CatalogEntity.java index da8c614d35d..b035578cd70 100644 --- a/core/src/main/java/org/apache/gravitino/meta/CatalogEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/CatalogEntity.java @@ -146,6 +146,11 @@ public CatalogInfo toCatalogInfoWithoutHiddenProps(Set hiddenKeys) { id, name, type, provider, comment, filteredProperties, auditInfo, namespace); } + public CatalogInfo toCatalogInfoWithResolvedProps(Map resolvedProperties) { + return new CatalogInfo( + id, name, type, provider, comment, resolvedProperties, auditInfo, namespace); + } + /** Builder class for creating instances of {@link CatalogEntity}. */ public static class Builder { diff --git a/core/src/test/java/org/apache/gravitino/TestCatalog.java b/core/src/test/java/org/apache/gravitino/TestCatalog.java index 8950ee23996..bdb409f20fb 100644 --- a/core/src/test/java/org/apache/gravitino/TestCatalog.java +++ b/core/src/test/java/org/apache/gravitino/TestCatalog.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.connector.BaseCatalog; -import org.apache.gravitino.connector.BasePropertiesMetadata; +import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.PropertiesMetadata; import org.apache.gravitino.connector.PropertyEntry; @@ -68,7 +68,7 @@ public PropertiesMetadata tablePropertiesMetadata() throws UnsupportedOperationE @Override public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperationException { - return new BasePropertiesMetadata() { + return new BaseCatalogPropertiesMetadata() { @Override protected Map> specificPropertyEntries() { return ImmutableMap.>builder() @@ -115,15 +115,6 @@ protected Map> specificPropertyEntries() { false, false, false)) - .put( - AUTHORIZATION_PROVIDER, - PropertyEntry.stringImmutablePropertyEntry( - Catalog.AUTHORIZATION_PROVIDER, - "The name of the authorization provider for Gravitino", - false, - null, - false, - false)) .build(); } }; diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java index 730f94fb9b8..007ed7e84c9 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java @@ -35,6 +35,7 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.meta.AuditInfo; @@ -454,6 +455,12 @@ public void testDropCatalog() { catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, provider, comment, props); // Test drop catalog + Exception exception = + Assertions.assertThrows( + CatalogInUseException.class, () -> catalogManager.dropCatalog(ident)); + Assertions.assertTrue(exception.getMessage().contains("Catalog metalake.test41 is in use")); + + Assertions.assertDoesNotThrow(() -> catalogManager.disableCatalog(ident)); boolean dropped = catalogManager.dropCatalog(ident); Assertions.assertTrue(dropped); diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java index 16e8b40ea3e..5f7ef050e22 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestSchemaOperationDispatcher.java @@ -23,6 +23,7 @@ import static org.apache.gravitino.TestBasePropertiesMetadata.COMMENT_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -151,7 +152,9 @@ public void testCreateAndLoadSchema() throws IOException { Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, loadedSchema.auditInfo().creator()); // Case 2: Test if the schema is not found in entity store - doThrow(new NoSuchEntityException("mock error")).when(entityStore).get(any(), any(), any()); + doThrow(new NoSuchEntityException("mock error")) + .when(entityStore) + .get(any(), eq(Entity.EntityType.SCHEMA), any()); entityStore.delete(schemaIdent, Entity.EntityType.SCHEMA); Schema loadedSchema1 = dispatcher.loadSchema(schemaIdent); Assertions.assertEquals(schema.name(), loadedSchema1.name()); @@ -165,7 +168,7 @@ public void testCreateAndLoadSchema() throws IOException { // Case 3: Test if entity store is failed to get the schema entity reset(entityStore); - doThrow(new IOException()).when(entityStore).get(any(), any(), any()); + doThrow(new IOException()).when(entityStore).get(any(), eq(Entity.EntityType.SCHEMA), any()); entityStore.delete(schemaIdent, Entity.EntityType.SCHEMA); Schema loadedSchema2 = dispatcher.loadSchema(schemaIdent); // Succeed to import the topic entity @@ -189,7 +192,7 @@ public void testCreateAndLoadSchema() throws IOException { .withCreateTime(Instant.now()) .build()) .build(); - doReturn(unmatchedEntity).when(entityStore).get(any(), any(), any()); + doReturn(unmatchedEntity).when(entityStore).get(any(), eq(Entity.EntityType.SCHEMA), any()); Schema loadedSchema3 = dispatcher.loadSchema(schemaIdent); // Succeed to import the schema entity reset(entityStore); diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java index f31b95e1e78..29eb655a354 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java @@ -27,6 +27,7 @@ import static org.apache.gravitino.TestBasePropertiesMetadata.COMMENT_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -45,6 +46,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.gravitino.Config; +import org.apache.gravitino.Entity; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -195,7 +197,9 @@ public void testCreateAndLoadTable() throws IOException { reset(entityStore); entityStore.delete(tableIdent1, TABLE); entityStore.delete(NameIdentifier.of(tableNs.levels()), SCHEMA); - doThrow(new NoSuchEntityException("")).when(entityStore).get(any(), any(), any()); + doThrow(new NoSuchEntityException("")) + .when(entityStore) + .get(any(), eq(Entity.EntityType.TABLE), any()); Table loadedTable2 = tableOperationDispatcher.loadTable(tableIdent1); // Succeed to import the topic entity Assertions.assertTrue(entityStore.exists(NameIdentifier.of(tableNs.levels()), SCHEMA)); @@ -207,7 +211,7 @@ public void testCreateAndLoadTable() throws IOException { reset(entityStore); entityStore.delete(tableIdent1, TABLE); entityStore.delete(NameIdentifier.of(tableNs.levels()), SCHEMA); - doThrow(new IOException()).when(entityStore).get(any(), any(), any()); + doThrow(new IOException()).when(entityStore).get(any(), eq(Entity.EntityType.TABLE), any()); Table loadedTable3 = tableOperationDispatcher.loadTable(tableIdent1); // Succeed to import the topic entity Assertions.assertTrue(entityStore.exists(NameIdentifier.of(tableNs.levels()), SCHEMA)); @@ -225,7 +229,7 @@ public void testCreateAndLoadTable() throws IOException { .withAuditInfo( AuditInfo.builder().withCreator("gravitino").withCreateTime(Instant.now()).build()) .build(); - doReturn(tableEntity).when(entityStore).get(any(), any(), any()); + doReturn(tableEntity).when(entityStore).get(any(), eq(Entity.EntityType.TABLE), any()); Table loadedTable4 = tableOperationDispatcher.loadTable(tableIdent1); // Succeed to import the topic entity reset(entityStore); diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java index ac694883d59..ac6b3bea4f4 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestTopicOperationDispatcher.java @@ -26,6 +26,7 @@ import static org.apache.gravitino.TestBasePropertiesMetadata.COMMENT_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -127,7 +128,9 @@ public void testCreateAndLoadTopic() throws IOException { reset(entityStore); entityStore.delete(topicIdent1, Entity.EntityType.TOPIC); entityStore.delete(NameIdentifier.of(topicNs.levels()), SCHEMA); - doThrow(new NoSuchEntityException("")).when(entityStore).get(any(), any(), any()); + doThrow(new NoSuchEntityException("")) + .when(entityStore) + .get(any(), eq(Entity.EntityType.TOPIC), any()); Topic loadedTopic2 = topicOperationDispatcher.loadTopic(topicIdent1); // Succeed to import the topic entity Assertions.assertTrue(entityStore.exists(topicIdent1, Entity.EntityType.TOPIC)); @@ -139,7 +142,7 @@ public void testCreateAndLoadTopic() throws IOException { reset(entityStore); entityStore.delete(topicIdent1, Entity.EntityType.TOPIC); entityStore.delete(NameIdentifier.of(topicNs.levels()), SCHEMA); - doThrow(new IOException()).when(entityStore).get(any(), any(), any()); + doThrow(new IOException()).when(entityStore).get(any(), eq(Entity.EntityType.TOPIC), any()); Topic loadedTopic3 = topicOperationDispatcher.loadTopic(topicIdent1); // Succeed to import the topic entity Assertions.assertTrue(entityStore.exists(NameIdentifier.of(topicNs.levels()), SCHEMA)); @@ -157,7 +160,7 @@ public void testCreateAndLoadTopic() throws IOException { .withAuditInfo( AuditInfo.builder().withCreator("gravitino").withCreateTime(Instant.now()).build()) .build(); - doReturn(unmatchedEntity).when(entityStore).get(any(), any(), any()); + doReturn(unmatchedEntity).when(entityStore).get(any(), eq(Entity.EntityType.TOPIC), any()); Topic loadedTopic4 = topicOperationDispatcher.loadTopic(topicIdent1); // Succeed to import the topic entity reset(entityStore); diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestCatalogEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestCatalogEvent.java index d2050894368..7b7cb9e3c46 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestCatalogEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestCatalogEvent.java @@ -19,6 +19,7 @@ package org.apache.gravitino.listener.api.event; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -243,7 +244,7 @@ private CatalogDispatcher mockCatalogDispatcher() { any(Map.class))) .thenReturn(catalog); when(dispatcher.loadCatalog(any(NameIdentifier.class))).thenReturn(catalog); - when(dispatcher.dropCatalog(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.dropCatalog(any(NameIdentifier.class), anyBoolean())).thenReturn(true); when(dispatcher.listCatalogs(any(Namespace.class))).thenReturn(null); when(dispatcher.alterCatalog(any(NameIdentifier.class), any(CatalogChange.class))) .thenReturn(catalog); diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java index eb0ff32e792..7693e5d4c9a 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java @@ -134,7 +134,7 @@ public Catalog createCatalog( * @return boolean */ public boolean dropCatalog(String catalogName) { - return metalake.dropCatalog(catalogName); + return metalake.dropCatalog(catalogName, true); } /** diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java index 9cdac8be7db..333aa83f0b6 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java @@ -79,7 +79,7 @@ static void hiveStartUp() { @AfterAll static void hiveStop() { Preconditions.checkNotNull(metalake); - metalake.dropCatalog(DEFAULT_HIVE_CATALOG); + metalake.dropCatalog(DEFAULT_HIVE_CATALOG, true); } protected static void initDefaultHiveCatalog() { diff --git a/server/src/main/java/org/apache/gravitino/server/web/Utils.java b/server/src/main/java/org/apache/gravitino/server/web/Utils.java index 86d2f8d1bad..69ec64daa08 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/Utils.java +++ b/server/src/main/java/org/apache/gravitino/server/web/Utils.java @@ -122,6 +122,28 @@ public static Response alreadyExists(String type, String message, Throwable thro .build(); } + public static Response notInUse(String message, Throwable throwable) { + return notInUse(throwable.getClass().getSimpleName(), message, throwable); + } + + public static Response notInUse(String type, String message, Throwable throwable) { + return Response.status(Response.Status.CONFLICT) + .entity(ErrorResponse.notInUse(type, message, throwable)) + .type(MediaType.APPLICATION_JSON) + .build(); + } + + public static Response inUse(String message, Throwable throwable) { + return inUse(throwable.getClass().getSimpleName(), message, throwable); + } + + public static Response inUse(String type, String message, Throwable throwable) { + return Response.status(Response.Status.CONFLICT) + .entity(ErrorResponse.inUse(type, message, throwable)) + .type(MediaType.APPLICATION_JSON) + .build(); + } + public static Response nonEmpty(String type, String message) { return nonEmpty(type, message, null); } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/CatalogOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/CatalogOperations.java index ebb1813e28e..ba934d5bdcf 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/CatalogOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/CatalogOperations.java @@ -26,6 +26,7 @@ import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.PATCH; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; @@ -41,6 +42,7 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.dto.requests.CatalogCreateRequest; +import org.apache.gravitino.dto.requests.CatalogSetRequest; import org.apache.gravitino.dto.requests.CatalogUpdateRequest; import org.apache.gravitino.dto.requests.CatalogUpdatesRequest; import org.apache.gravitino.dto.responses.BaseResponse; @@ -187,6 +189,52 @@ public Response testConnection( } } + @PATCH + @Path("{catalog}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "set-catalog." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "set-catalog", absolute = true) + public Response setCatalog( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalogName, + CatalogSetRequest request) { + LOG.info("Received set request for catalog: {}.{}", metalake, catalogName); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifierUtil.ofCatalog(metalake, catalogName); + TreeLockUtils.doWithTreeLock( + NameIdentifierUtil.ofMetalake(metalake), + LockType.WRITE, + () -> { + if (request.isInUse()) { + catalogDispatcher.enableCatalog(ident); + } else { + catalogDispatcher.disableCatalog(ident); + } + return null; + }); + Response response = Utils.ok(new BaseResponse()); + LOG.info( + "Successfully {} catalog: {}.{}", + request.isInUse() ? "enable" : "disable", + metalake, + catalogName); + return response; + }); + + } catch (Exception e) { + LOG.info( + "Failed to {} catalog: {}.{}", + request.isInUse() ? "enable" : "disable", + metalake, + catalogName); + return ExceptionHandlers.handleCatalogException( + OperationType.ENABLE, catalogName, metalake, e); + } + } + @GET @Path("{catalog}") @Produces("application/vnd.gravitino.v1+json") @@ -252,7 +300,9 @@ public Response alterCatalog( @Timed(name = "drop-catalog." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "drop-catalog", absolute = true) public Response dropCatalog( - @PathParam("metalake") String metalakeName, @PathParam("catalog") String catalogName) { + @PathParam("metalake") String metalakeName, + @PathParam("catalog") String catalogName, + @DefaultValue("false") @QueryParam("force") boolean force) { LOG.info("Received drop catalog request for catalog: {}.{}", metalakeName, catalogName); try { return Utils.doAs( @@ -263,7 +313,7 @@ public Response dropCatalog( TreeLockUtils.doWithTreeLock( NameIdentifierUtil.ofMetalake(metalakeName), LockType.WRITE, - () -> catalogDispatcher.dropCatalog(ident)); + () -> catalogDispatcher.dropCatalog(ident, force)); if (!dropped) { LOG.warn("Failed to drop catalog {} under metalake {}", catalogName, metalakeName); } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index dd8c21ab4cb..2fe844d8c02 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -24,6 +24,8 @@ import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; +import org.apache.gravitino.exceptions.CatalogInUseException; +import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.ConnectionFailedException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.ForbiddenException; @@ -178,6 +180,9 @@ public Response handle(OperationType op, String partition, String table, Excepti } else if (e instanceof UnsupportedOperationException) { return Utils.unsupportedOperation(errorMsg, e); + } else if (e instanceof CatalogNotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, partition, table, e); } @@ -216,6 +221,9 @@ public Response handle(OperationType op, String table, String schema, Exception } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); + } else if (e instanceof CatalogNotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, table, schema, e); } @@ -257,6 +265,9 @@ public Response handle(OperationType op, String schema, String catalog, Exceptio } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); + } else if (e instanceof CatalogNotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, schema, catalog, e); } @@ -295,6 +306,12 @@ public Response handle(OperationType op, String catalog, String metalake, Except } else if (e instanceof CatalogAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof CatalogNotInUseException) { + return Utils.notInUse(errorMsg, e); + + } else if (e instanceof CatalogInUseException) { + return Utils.inUse(errorMsg, e); + } else { return super.handle(op, catalog, metalake, e); } @@ -360,6 +377,9 @@ public Response handle(OperationType op, String fileset, String schema, Exceptio } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); + } else if (e instanceof CatalogNotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, fileset, schema, e); } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java index 37032124fa1..8d4bc322ae7 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java @@ -24,6 +24,8 @@ public enum OperationType { LOAD, ALTER, DROP, + ENABLE, + DISABLE, /** This is a special operation type that is used to get a partition from a table. */ GET, ADD, diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalog.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalog.java index bcd5107b4fd..dbe94f287f5 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalog.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalog.java @@ -18,7 +18,8 @@ */ package org.apache.gravitino.server.web.rest; -import com.google.common.collect.ImmutableMap; +import static org.apache.gravitino.connector.BaseCatalogPropertiesMetadata.BASIC_CATALOG_PROPERTIES_METADATA; + import java.io.IOException; import java.util.Map; import org.apache.gravitino.NameIdentifier; @@ -29,7 +30,6 @@ import org.apache.gravitino.connector.PropertiesMetadata; public class TestCatalog extends BaseCatalog { - private static final PropertiesMetadata PROPERTIES_METADATA = ImmutableMap::of; @Override public String shortName() { @@ -61,6 +61,6 @@ public void close() throws IOException {} @Override public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperationException { - return PROPERTIES_METADATA; + return BASIC_CATALOG_PROPERTIES_METADATA; } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalogOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalogOperations.java index 98ac3ed4040..9b8c8609336 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalogOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestCatalogOperations.java @@ -19,10 +19,12 @@ package org.apache.gravitino.server.web.rest; import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; +import static org.apache.gravitino.Catalog.PROPERTY_IN_USE; import static org.apache.gravitino.Configs.TREE_LOCK_CLEAN_INTERVAL; import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -48,6 +50,7 @@ import org.apache.gravitino.catalog.CatalogManager; import org.apache.gravitino.dto.CatalogDTO; import org.apache.gravitino.dto.requests.CatalogCreateRequest; +import org.apache.gravitino.dto.requests.CatalogSetRequest; import org.apache.gravitino.dto.requests.CatalogUpdateRequest; import org.apache.gravitino.dto.requests.CatalogUpdatesRequest; import org.apache.gravitino.dto.responses.BaseResponse; @@ -65,6 +68,7 @@ import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.rest.RESTUtils; import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.test.JerseyTest; import org.glassfish.jersey.test.TestProperties; @@ -184,13 +188,15 @@ public void testListCatalogsInfo() { Assertions.assertEquals("catalog1", catalogDTO1.name()); Assertions.assertEquals(Catalog.Type.RELATIONAL, catalogDTO1.type()); Assertions.assertEquals("comment", catalogDTO1.comment()); - Assertions.assertEquals(ImmutableMap.of("key", "value"), catalogDTO1.properties()); + Assertions.assertEquals( + ImmutableMap.of("key", "value", PROPERTY_IN_USE, "true"), catalogDTO1.properties()); CatalogDTO catalogDTO2 = catalogDTOs[1]; Assertions.assertEquals("catalog2", catalogDTO2.name()); Assertions.assertEquals(Catalog.Type.RELATIONAL, catalogDTO2.type()); Assertions.assertEquals("comment", catalogDTO2.comment()); - Assertions.assertEquals(ImmutableMap.of("key", "value"), catalogDTO2.properties()); + Assertions.assertEquals( + ImmutableMap.of("key", "value", PROPERTY_IN_USE, "true"), catalogDTO2.properties()); doThrow(new NoSuchMetalakeException("mock error")).when(manager).listCatalogsInfo(any()); Response resp1 = @@ -237,7 +243,8 @@ public void testCreateCatalog() { Assertions.assertEquals("catalog1", catalogDTO.name()); Assertions.assertEquals(Catalog.Type.RELATIONAL, catalogDTO.type()); Assertions.assertEquals("comment", catalogDTO.comment()); - Assertions.assertEquals(ImmutableMap.of("key", "value"), catalogDTO.properties()); + Assertions.assertEquals( + ImmutableMap.of("key", "value", PROPERTY_IN_USE, "true"), catalogDTO.properties()); // Test throw NoSuchMetalakeException doThrow(new NoSuchMetalakeException("mock error")) @@ -351,7 +358,8 @@ public void testLoadCatalog() { Assertions.assertEquals("catalog1", catalogDTO.name()); Assertions.assertEquals(Catalog.Type.RELATIONAL, catalogDTO.type()); Assertions.assertEquals("comment", catalogDTO.comment()); - Assertions.assertEquals(ImmutableMap.of("key", "value"), catalogDTO.properties()); + Assertions.assertEquals( + ImmutableMap.of("key", "value", PROPERTY_IN_USE, "true"), catalogDTO.properties()); // Test throw NoSuchMetalakeException doThrow(new NoSuchMetalakeException("mock error")).when(manager).loadCatalog(any()); @@ -420,7 +428,8 @@ public void testAlterCatalog() { Assertions.assertEquals("catalog2", catalogDTO.name()); Assertions.assertEquals(Catalog.Type.RELATIONAL, catalogDTO.type()); Assertions.assertEquals("comment", catalogDTO.comment()); - Assertions.assertEquals(ImmutableMap.of("key", "value"), catalogDTO.properties()); + Assertions.assertEquals( + ImmutableMap.of("key", "value", PROPERTY_IN_USE, "true"), catalogDTO.properties()); // Test throw NoSuchCatalogException doThrow(new NoSuchCatalogException("mock error")).when(manager).alterCatalog(any(), any()); @@ -468,7 +477,7 @@ public void testAlterCatalog() { @Test public void testDropCatalog() { - when(manager.dropCatalog(any())).thenReturn(true); + when(manager.dropCatalog(any(), anyBoolean())).thenReturn(true); Response resp = target("/metalakes/metalake1/catalogs/catalog1") @@ -481,8 +490,8 @@ public void testDropCatalog() { Assertions.assertEquals(0, dropResponse.getCode()); Assertions.assertTrue(dropResponse.dropped()); - // Test when failed to drop catalog - when(manager.dropCatalog(any())).thenReturn(false); + // Test catalog does not exist + when(manager.dropCatalog(any(), anyBoolean())).thenReturn(false); Response resp2 = target("/metalakes/metalake1/catalogs/catalog1") @@ -496,7 +505,7 @@ public void testDropCatalog() { Assertions.assertFalse(dropResponse2.dropped()); // Test throw internal RuntimeException - doThrow(new RuntimeException("mock error")).when(manager).dropCatalog(any()); + doThrow(new RuntimeException("mock error")).when(manager).dropCatalog(any(), anyBoolean()); Response resp3 = target("/metalakes/metalake1/catalogs/catalog1") .request(MediaType.APPLICATION_JSON_TYPE) @@ -510,6 +519,37 @@ public void testDropCatalog() { Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); } + @Test + public void testSetCatalog() { + CatalogSetRequest req = new CatalogSetRequest(true); + doNothing().when(manager).enableCatalog(any()); + + Response resp = + target("/metalakes/metalake1/catalogs/catalog1") + .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .method("PATCH", Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + BaseResponse baseResponse = resp.readEntity(BaseResponse.class); + Assertions.assertEquals(0, baseResponse.getCode()); + + req = new CatalogSetRequest(false); + doNothing().when(manager).disableCatalog(any()); + + resp = + target("/metalakes/metalake1/catalogs/catalog1") + .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .method("PATCH", Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + baseResponse = resp.readEntity(BaseResponse.class); + Assertions.assertEquals(0, baseResponse.getCode()); + } + private static TestCatalog buildCatalog(String metalake, String catalogName) { CatalogEntity entity = CatalogEntity.builder() diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java index 162fd10b746..466932b0608 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoConnectorIT.java @@ -1372,6 +1372,7 @@ void testDropCatalogAndCreateAgain() { boolean success = checkTrinoHasLoaded(sql, 30); Assertions.assertTrue(success, "Trino should load the catalog: " + sql); + createdMetalake.disableCatalog(catalogName); createdMetalake.dropCatalog(catalogName); // We need to test we can't load this catalog any more by Trino. success = checkTrinoHasRemoved(sql, 30); diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java index 14558237786..eabd7732194 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java @@ -232,6 +232,7 @@ protected static void dropCatalog(String catalogName) { LOG.info("Drop schema \"{}.{}\".{}", metalakeName, catalogName, schema); }); + metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); LOG.info("Drop catalog \"{}.{}\"", metalakeName, catalogName); } diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt index be0a71f3c77..39d2523a192 100644 --- a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt +++ b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00008_alter_catalog.txt @@ -1,17 +1,17 @@ CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino""}" +"gt_mysql_xxx1","jdbc-mysql","{""in-use"":""true"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino""}" CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""test_key"":""test_value"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" +"gt_mysql_xxx1","jdbc-mysql","{""in-use"":""true"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""test_key"":""test_value"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""test_key"":""test_value""}" +"gt_mysql_xxx1","jdbc-mysql","{""in-use"":""true"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""test_key"":""test_value""}" CALL -"gt_mysql_xxx1","jdbc-mysql","{""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" +"gt_mysql_xxx1","jdbc-mysql","{""in-use"":""true"",""jdbc-driver"":""com.mysql.cj.jdbc.Driver"",""jdbc-password"":""ds123"",""jdbc-url"":""jdbc:mysql://%/?useSSL=false"",""jdbc-user"":""trino"",""trino.bypass.join-pushdown.strategy"":""EAGER""}" CALL diff --git a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java index 7b195b642a7..7b7c6144ba8 100644 --- a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java +++ b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/storedprocdure/DropCatalogStoredProcedure.java @@ -79,12 +79,7 @@ public void dropCatalog(String catalogName, boolean ignoreNotExist) { GravitinoErrorCode.GRAVITINO_CATALOG_NOT_EXISTS, "Catalog " + NameIdentifier.of(metalake, catalogName) + " not exists."); } - boolean dropped = catalogConnector.getMetalake().dropCatalog(catalogName); - if (!dropped) { - throw new TrinoException( - GravitinoErrorCode.GRAVITINO_UNSUPPORTED_OPERATION, - "Failed to drop no empty catalog " + catalogName); - } + catalogConnector.getMetalake().dropCatalog(catalogName, true); catalogConnectorManager.loadMetalakeSync(); diff --git a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java index 8cfb157b1b3..05ff602c180 100644 --- a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java @@ -164,7 +164,7 @@ public Catalog answer(InvocationOnMock invocation) throws Throwable { } }); - when(metaLake.dropCatalog(anyString())) + when(metaLake.dropCatalog(anyString(), anyBoolean())) .thenAnswer( new Answer() { @Override diff --git a/web/web/src/lib/api/catalogs/index.js b/web/web/src/lib/api/catalogs/index.js index 30e08dea2ea..5272d50f7d0 100644 --- a/web/web/src/lib/api/catalogs/index.js +++ b/web/web/src/lib/api/catalogs/index.js @@ -27,7 +27,7 @@ const Apis = { UPDATE: ({ metalake, catalog }) => `/api/metalakes/${encodeURIComponent(metalake)}/catalogs/${encodeURIComponent(catalog)}`, DELETE: ({ metalake, catalog }) => - `/api/metalakes/${encodeURIComponent(metalake)}/catalogs/${encodeURIComponent(catalog)}` + `/api/metalakes/${encodeURIComponent(metalake)}/catalogs/${encodeURIComponent(catalog)}?force=true` } export const getCatalogsApi = params => {