Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[KYUUBI #4171] Support skip retrieving table's properties to speed up GetTables operation #4444

Closed
wants to merge 3 commits into from
Closed

[KYUUBI #4171] Support skip retrieving table's properties to speed up GetTables operation #4444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5e8687c
Supports ignore table comment when list all tables.
liaoyt Mar 4, 2023
0c9985e
add doc
liaoyt Mar 4, 2023
af5e60e
rename config
liaoyt Mar 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 18 additions & 17 deletions docs/deployment/settings.md
View file
Open in desktop

Large diffs are not rendered by default.

15 changes: 14 additions & 1 deletion ...-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package org.apache.kyuubi.engine.spark.operation

import org.apache.spark.sql.types.StructType

import org.apache.kyuubi.config.KyuubiConf.OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES
import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim
import org.apache.kyuubi.operation.IterableFetchIterator
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
Expand All @@ -32,6 +33,12 @@ class GetTables(
tableTypes: Set[String])
extends SparkOperation(session) {

protected val ignoreTableProperties =
spark.conf.getOption(OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES.key) match {
case Some(s) => s.toBoolean
case _ => session.sessionManager.getConf.get(OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES)
}

override def statement: String = {
super.statement +
s" [catalog: $catalog," +
Expand Down Expand Up @@ -68,7 +75,13 @@ class GetTables(
val tablePattern = toJavaRegex(tableName)
val sparkShim = SparkCatalogShim()
val catalogTablesAndViews =
sparkShim.getCatalogTablesOrViews(spark, catalog, schemaPattern, tablePattern, tableTypes)
sparkShim.getCatalogTablesOrViews(
spark,
catalog,
schemaPattern,
tablePattern,
tableTypes,
ignoreTableProperties)

val allTableAndViews =
if (tableTypes.exists("VIEW".equalsIgnoreCase)) {
Expand Down
3 changes: 2 additions & 1 deletion ...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ class CatalogShim_v2_4 extends SparkCatalogShim {
catalogName: String,
schemaPattern: String,
tablePattern: String,
tableTypes: Set[String]): Seq[Row] = {
tableTypes: Set[String],
ignoreTableProperties: Boolean): Seq[Row] = {
val catalog = spark.sessionState.catalog
val databases = catalog.listDatabases(schemaPattern)

Expand Down
10 changes: 6 additions & 4 deletions ...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 {
catalogName: String,
schemaPattern: String,
tablePattern: String,
tableTypes: Set[String]): Seq[Row] = {
tableTypes: Set[String],
ignoreTableProperties: Boolean = false): Seq[Row] = {
val catalog = getCatalog(spark, catalogName)
val namespaces = listNamespacesWithPattern(catalog, schemaPattern)
catalog match {
Expand All @@ -160,16 +161,17 @@ class CatalogShim_v3_0 extends CatalogShim_v2_4 {
SESSION_CATALOG,
schemaPattern,
tablePattern,
tableTypes)
tableTypes,
ignoreTableProperties)
case tc: TableCatalog =>
val tp = tablePattern.r.pattern
val identifiers = namespaces.flatMap { ns =>
tc.listTables(ns).filter(i => tp.matcher(quoteIfNeeded(i.name())).matches())
}
identifiers.map { ident =>
val table = tc.loadTable(ident)
// TODO: restore view type for session catalog
val comment = table.properties().getOrDefault(TableCatalog.PROP_COMMENT, "")
val comment = if (ignoreTableProperties) ""
else tc.loadTable(ident).properties().getOrDefault(TableCatalog.PROP_COMMENT, "")
val schema = ident.namespace().map(quoteIfNeeded).mkString(".")
val tableName = quoteIfNeeded(ident.name())
Row(catalog.name(), schema, tableName, "TABLE", comment, null, null, null, null, null)
Expand Down
3 changes: 2 additions & 1 deletion ...park-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ trait SparkCatalogShim extends Logging {
catalogName: String,
schemaPattern: String,
tablePattern: String,
tableTypes: Set[String]): Seq[Row]
tableTypes: Set[String],
ignoreTableProperties: Boolean): Seq[Row]

def getTempViews(
spark: SparkSession,
Expand Down
7 changes: 7 additions & 0 deletions kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2713,4 +2713,11 @@ object KyuubiConf {
.version("1.7.0")
.timeConf
.createWithDefault(Duration.ofSeconds(60).toMillis)

val OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES: ConfigEntry[Boolean] =
buildConf("kyuubi.operation.getTables.ignoreTableProperties")
.doc("Speed up the `GetTables` operation by returning table identities only.")
.version("1.8.0")
.booleanConf
.createWithDefault(false)
}