[apache#4089] fix: the problem of slow acquisition of hive table list

mygrsun · Oct 31, 2024 · 0d58522 · 0d58522
1 parent a2f7093
commit 0d58522
Showing 1 changed file with 13 additions and 35 deletions.
diff --git a/...s/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java b/...s/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveCatalogOperations.java
@@ -21,9 +21,7 @@
 import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta.LIST_ALL_TABLES;
 import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta.METASTORE_URIS;
 import static org.apache.gravitino.catalog.hive.HiveCatalogPropertiesMeta.PRINCIPAL;
-import static org.apache.gravitino.catalog.hive.HiveTable.ICEBERG_TABLE_TYPE_VALUE;
 import static org.apache.gravitino.catalog.hive.HiveTable.SUPPORT_TABLE_TYPES;
-import static org.apache.gravitino.catalog.hive.HiveTable.TABLE_TYPE_PROP;
 import static org.apache.gravitino.catalog.hive.HiveTablePropertiesMetadata.COMMENT;
 import static org.apache.gravitino.catalog.hive.HiveTablePropertiesMetadata.TABLE_TYPE;
 import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
@@ -93,6 +91,7 @@
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.thrift.TException;
@@ -539,23 +538,18 @@ public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaExcep
       // then based on
       // those names we can obtain metadata for each individual table and get the type we needed.
       List<String> allTables = clientPool.run(c -> c.getAllTables(schemaIdent.name()));
-      return clientPool.run(
-          c ->
-              c.getTableObjectsByName(schemaIdent.name(), allTables).stream()
-                  .filter(
-                      tb -> {
-                        boolean isSupportTable = SUPPORT_TABLE_TYPES.contains(tb.getTableType());
-                        if (!isSupportTable) {
-                          return false;
-                        }
-                        if (!listAllTables) {
-                          Map<String, String> parameters = tb.getParameters();
-                          return isHiveTable(parameters);
-                        }
-                        return true;
-                      })
-                  .map(tb -> NameIdentifier.of(namespace, tb.getTableName()))
-                  .toArray(NameIdentifier[]::new));
+      if (!listAllTables) {
+        String filter =
+            String.format(
+                "%stable_type = \"ICEBERG\"", hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS);
+        List<String> allTables_iceberg =
+            clientPool.run(c -> c.listTableNamesByFilter(schemaIdent.name(), filter, (short) -1));
+        allTables.removeAll(allTables_iceberg);
+      }
+      return allTables.stream()
+          .map(tbName -> NameIdentifier.of(namespace, tbName))
+          .toArray(NameIdentifier[]::new);
+
     } catch (UnknownDBException e) {
       throw new NoSuchSchemaException(
           "Schema (database) does not exist %s in Hive Metastore", namespace);
@@ -569,22 +563,6 @@ public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaExcep
     }
   }
 
-  boolean isHiveTable(Map<String, String> tableParameters) {
-    if (isIcebergTable(tableParameters)) return false;
-    return true;
-  }
-
-  boolean isIcebergTable(Map<String, String> tableParameters) {
-    if (tableParameters != null) {
-      boolean isIcebergTable =
-          ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(tableParameters.get(TABLE_TYPE_PROP));
-      if (isIcebergTable) {
-        return true;
-      }
-    }
-    return false;
-  }
-
   /**
    * Loads a table from the Hive Metastore.
    *