rename config

apache · Mar 5, 2023 · af5e60e · af5e60e
1 parent 0c9985e
commit af5e60e
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 30 deletions.
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
diff --git a/...-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala b/...-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala
@@ -19,6 +19,7 @@ package org.apache.kyuubi.engine.spark.operation
 
 import org.apache.spark.sql.types.StructType
 
+import org.apache.kyuubi.config.KyuubiConf.OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES
 import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
 import org.apache.kyuubi.operation.IterableFetchIterator
 import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
@@ -32,6 +33,12 @@ class GetTables(
     tableTypes: Set[String])
   extends SparkOperation(session) {
 
+  protected val ignoreTableProperties =
+    spark.conf.getOption(OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES.key) match {
+      case Some(s) => s.toBoolean
+      case _ => session.sessionManager.getConf.get(OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES)
+    }
+
   override def statement: String = {
     super.statement +
       s" [catalog: $catalog," +
@@ -68,7 +75,13 @@ class GetTables(
       val tablePattern = toJavaRegex(tableName)
       val sparkShim = SparkCatalogShim()
       val catalogTablesAndViews =
-        sparkShim.getCatalogTablesOrViews(spark, catalog, schemaPattern, tablePattern, tableTypes)
+        sparkShim.getCatalogTablesOrViews(
+          spark,
+          catalog,
+          schemaPattern,
+          tablePattern,
+          tableTypes,
+          ignoreTableProperties)
 
       val allTableAndViews =
         if (tableTypes.exists("VIEW".equalsIgnoreCase)) {

diff --git a/...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala b/...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala
@@ -64,7 +64,8 @@ class CatalogShim_v2_4 extends SparkCatalogShim {
       catalogName: String,
       schemaPattern: String,
       tablePattern: String,
-      tableTypes: Set[String]): Seq[Row] = {
+      tableTypes: Set[String],
+      ignoreTableProperties: Boolean): Seq[Row] = {
     val catalog = spark.sessionState.catalog
     val databases = catalog.listDatabases(schemaPattern)
 

diff --git a/...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala b/...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala
@@ -22,7 +22,6 @@ import java.util.regex.Pattern
 import org.apache.spark.sql.{Row, SparkSession}
 import org.apache.spark.sql.connector.catalog.{CatalogExtension, CatalogPlugin, SupportsNamespaces, TableCatalog}
 
-import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim.SESSION_CATALOG
 
 class CatalogShim_v3_0 extends CatalogShim_v2_4 {
@@ -151,7 +150,8 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 {
       catalogName: String,
       schemaPattern: String,
       tablePattern: String,
-      tableTypes: Set[String]): Seq[Row] = {
+      tableTypes: Set[String],
+      ignoreTableProperties: Boolean = false): Seq[Row] = {
     val catalog = getCatalog(spark, catalogName)
     val namespaces = listNamespacesWithPattern(catalog, schemaPattern)
     catalog match {
@@ -161,17 +161,16 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 {
           SESSION_CATALOG,
           schemaPattern,
           tablePattern,
-          tableTypes)
+          tableTypes,
+          ignoreTableProperties)
       case tc: TableCatalog =>
         val tp = tablePattern.r.pattern
         val identifiers = namespaces.flatMap { ns =>
           tc.listTables(ns).filter(i => tp.matcher(quoteIfNeeded(i.name())).matches())
         }
-        val listTablesOnly = spark.conf.getOption(KyuubiConf.ENGINE_SPARK_LIST_TABLES.key)
-          .map(_.toBoolean).getOrElse(KyuubiConf.ENGINE_SPARK_LIST_TABLES.defaultVal.get)
         identifiers.map { ident =>
           // TODO: restore view type for session catalog
-          val comment = if (listTablesOnly) ""
+          val comment = if (ignoreTableProperties) ""
           else tc.loadTable(ident).properties().getOrDefault(TableCatalog.PROP_COMMENT, "")
           val schema = ident.namespace().map(quoteIfNeeded).mkString(".")
           val tableName = quoteIfNeeded(ident.name())

diff --git a/...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala b/...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala
@@ -69,7 +69,8 @@ trait SparkCatalogShim extends Logging {
       catalogName: String,
       schemaPattern: String,
       tablePattern: String,
-      tableTypes: Set[String]): Seq[Row]
+      tableTypes: Set[String],
+      ignoreTableProperties: Boolean): Seq[Row]
 
   def getTempViews(
       spark: SparkSession,

diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -2714,9 +2714,9 @@ object KyuubiConf {
       .timeConf
       .createWithDefault(Duration.ofSeconds(60).toMillis)
 
-  val ENGINE_SPARK_LIST_TABLES: ConfigEntry[Boolean] =
-    buildConf("kyuubi.engine.spark.list.tables")
-      .doc("Only query table identifiers when set to true. Work on Spark 3.x only.")
+  val OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES: ConfigEntry[Boolean] =
+    buildConf("kyuubi.operation.getTables.ignoreTableProperties")
+      .doc("Speed up the `GetTables` operation by returning table identities only.")
       .version("1.8.0")
       .booleanConf
       .createWithDefault(false)