From 907973e9540d82d4acf2c7915ceaeaccc8b497ab Mon Sep 17 00:00:00 2001 From: Marig_Weizhi Date: Tue, 31 Dec 2024 11:50:21 +0800 Subject: [PATCH] [hotfix] Fix drop database not working in terminal (#3363) * Implement dropNamespace * test namespace for SparkUnifiedCatalog * Implement cascade or non-cascading drop namespace in base * Restore private * fix * Remove redundant method --------- Co-authored-by: big face cat <731030576@qq.com> --- .../amoro/spark/SparkUnifiedCatalogBase.java | 16 ++++++++++++-- .../unified/UnifiedCatalogTestSuites.java | 14 +++++++++++++ .../amoro/spark/SparkUnifiedCatalog.java | 21 ------------------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalogBase.java b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalogBase.java index aed47f5b8a..3675520e3a 100644 --- a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalogBase.java +++ b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalogBase.java @@ -173,10 +173,17 @@ public void alterNamespace(String[] namespace, NamespaceChange... changes) { throw new UnsupportedOperationException("Cannot apply namespace change"); } - @Override - public boolean dropNamespace(String[] namespace) { + public boolean dropNamespace(String[] namespace, boolean cascade) + throws NoSuchNamespaceException { String database = namespaceToDatabase(namespace); + if (!unifiedCatalog.databaseExists(database)) { + throw new NoSuchNamespaceException(namespace); + } List tables = unifiedCatalog.listTables(database); + if (!tables.isEmpty() && !cascade) { + throw new IllegalStateException("Namespace '" + database + "' is non empty."); + } + for (TableIDWithFormat id : tables) { unifiedCatalog.dropTable(database, id.getIdentifier().getTableName(), true); } @@ -184,6 +191,11 @@ public boolean dropNamespace(String[] namespace) { return !unifiedCatalog.databaseExists(database); } + @Override + public boolean dropNamespace(String[] namespace) throws NoSuchNamespaceException { + return dropNamespace(namespace, false); + } + @Override public Identifier[] listTables(String[] namespace) throws NoSuchNamespaceException { String database = namespaceToDatabase(namespace); diff --git a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/unified/UnifiedCatalogTestSuites.java b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/unified/UnifiedCatalogTestSuites.java index 480cd54e19..d87fcf890e 100644 --- a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/unified/UnifiedCatalogTestSuites.java +++ b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/unified/UnifiedCatalogTestSuites.java @@ -91,6 +91,9 @@ public void testTableFormats(TableFormat format, boolean sessionCatalog) { long count = sql(sqlText).count(); Assertions.assertEquals(expect, count); + // create and drop namespace + testNamespaceWithSql(); + // visit sub tables. testVisitSubTable(format, sessionCatalog); @@ -108,6 +111,17 @@ public void testTableFormats(TableFormat format, boolean sessionCatalog) { Assertions.assertFalse(unifiedCatalog().tableExists(target().database, target().table)); } + private void testNamespaceWithSql() { + // Use SparkTestBase::sql method to test SparkUnifiedCatalog instead of CommonUnifiedCatalog. + String createDatabase = "CREATE DATABASE test_unified_catalog"; + sql(createDatabase); + Assertions.assertTrue(unifiedCatalog().databaseExists("test_unified_catalog")); + + String dropDatabase = "DROP DATABASE test_unified_catalog"; + sql(dropDatabase); + Assertions.assertFalse(unifiedCatalog().databaseExists("test_unified_catalog")); + } + private String pkDDL(TableFormat format) { if (TableFormat.MIXED_HIVE.equals(format) || TableFormat.MIXED_ICEBERG.equals(format)) { return ", primary key(id)"; diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalog.java b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalog.java index 09fbfeb5a0..dac09c5496 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalog.java +++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/src/main/java/org/apache/amoro/spark/SparkUnifiedCatalog.java @@ -22,7 +22,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException; import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; -import org.apache.spark.sql.catalyst.analysis.NonEmptyNamespaceException; import org.apache.spark.sql.connector.catalog.FunctionCatalog; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.SupportsNamespaces; @@ -69,26 +68,6 @@ public UnboundFunction loadFunction(Identifier ident) throws NoSuchFunctionExcep throw new NoSuchFunctionException(ident); } - /** - * Drop a namespace from the catalog with cascade mode, recursively dropping all objects within - * the namespace if cascade is true. - * - *

If the catalog implementation does not support this operation, it may throw {@link - * UnsupportedOperationException}. - * - * @param namespace a multi-part namespace - * @param cascade When true, deletes all objects under the namespace - * @return true if the namespace was dropped - * @throws NoSuchNamespaceException If the namespace does not exist (optional) - * @throws NonEmptyNamespaceException If the namespace is non-empty and cascade is false - * @throws UnsupportedOperationException If drop is not a supported operation - */ - @Override - public boolean dropNamespace(String[] namespace, boolean cascade) - throws NoSuchNamespaceException, NonEmptyNamespaceException { - return false; - } - /** * Load table metadata of a specific version by {@link Identifier identifier} from the catalog. *