Speed up GetTables operation

apache · Jan 25, 2024 · 615b747 · 615b747
1 parent 47a1091
commit 615b747
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 16 deletions.
diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md
@@ -392,7 +392,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
 
 |                       Key                        |                                     Default                                     |                                                                                                                                                                                                                                                 Meaning                                                                                                                                                                                                                                                  |   Type   | Since |
 |--------------------------------------------------|---------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------|
-| kyuubi.operation.getTables.ignoreTableProperties | false                                                                           | Speed up the `GetTables` operation by returning table identities only.                                                                                                                                                                                                                                                                                                                                                                                                                                   | boolean  | 1.8.0 |
+| kyuubi.operation.getTables.ignoreTableProperties | false                                                                           | Speed up the `GetTables` operation by ignoring `tableTypes` query criteria, and returning table identities only.                                                                                                                                                                                                                                                                                                                                                                                         | boolean  | 1.8.0 |
 | kyuubi.operation.idle.timeout                    | PT3H                                                                            | Operation will be closed when it's not accessed for this duration of time                                                                                                                                                                                                                                                                                                                                                                                                                                | duration | 1.0.0 |
 | kyuubi.operation.interrupt.on.cancel             | true                                                                            | When true, all running tasks will be interrupted if one cancels a query. When false, all running tasks will remain until finished.                                                                                                                                                                                                                                                                                                                                                                       | boolean  | 1.2.0 |
 | kyuubi.operation.language                        | SQL                                                                             | Choose a programing language for the following inputs<ul><li>SQL: (Default) Run all following statements as SQL queries.</li><li>SCALA: Run all following input as scala codes</li><li>PYTHON: (Experimental) Run all following input as Python codes with Spark engine</li></ul>                                                                                                                                                                                                                        | string   | 1.5.0 |

diff --git a/...ark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/util/SparkCatalogUtils.scala b/...ark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/util/SparkCatalogUtils.scala
@@ -173,21 +173,41 @@ object SparkCatalogUtils extends Logging {
 
         databases.flatMap { db =>
           val identifiers = catalog.listTables(db, tablePattern, includeLocalTempViews = false)
-          catalog.getTablesByName(identifiers)
-            .filter(t => isMatchedTableType(tableTypes, t.tableType.name)).map { t =>
-              val typ = if (t.tableType.name == VIEW) VIEW else TABLE
-              Row(
-                catalogName,
-                t.database,
-                t.identifier.table,
-                typ,
-                t.comment.getOrElse(""),
-                null,
-                null,
-                null,
-                null,
-                null)
+          if (ignoreTableProperties) {
+            identifiers.map {
+              case TableIdentifier(
+                    table: String,
+                    database: Option[String],
+                    catalog: Option[String]) =>
+                Row(
+                  catalog.getOrElse(catalogName),
+                  database.getOrElse("default"),
+                  table,
+                  TABLE, // ignore tableTypes criteria and simply treat all table type as TABLE
+                  "",
+                  null,
+                  null,
+                  null,
+                  null,
+                  null)
             }
+          } else {
+            catalog.getTablesByName(identifiers)
+              .filter(t => isMatchedTableType(tableTypes, t.tableType.name)).map { t =>
+                val typ = if (t.tableType.name == VIEW) VIEW else TABLE
+                Row(
+                  catalogName,
+                  t.database,
+                  t.identifier.table,
+                  typ,
+                  t.comment.getOrElse(""),
+                  null,
+                  null,
+                  null,
+                  null,
+                  null)
+              }
+          }
         }
       case tc: TableCatalog =>
         val tp = tablePattern.r.pattern

diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -3433,7 +3433,8 @@ object KyuubiConf {
 
   val OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES: ConfigEntry[Boolean] =
     buildConf("kyuubi.operation.getTables.ignoreTableProperties")
-      .doc("Speed up the `GetTables` operation by returning table identities only.")
+      .doc("Speed up the `GetTables` operation by ignoring `tableTypes` query criteria, " +
+        "and returning table identities only.")
       .version("1.8.0")
       .booleanConf
       .createWithDefault(false)