Skip to content

Commit

Permalink
comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@turboFei
turboFei committed Dec 2, 2024
1 parent f493a2a commit 4568c77
Show file tree
Hide file tree
Showing 7 changed files with 18 additions and 26 deletions.
2 changes: 1 addition & 1 deletion docs/configuration/settings.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.metadata.request.async.retry.queue.size | 65536 | The maximum queue size for buffering metadata requests in memory when the external metadata storage is down. Requests will be dropped if the queue exceeds. Only take affect when kyuubi.metadata.request.async.retry.enabled is `true`. | int | 1.6.0 |
| kyuubi.metadata.request.async.retry.threads | 10 | Number of threads in the metadata request async retry manager thread pool. Only take affect when kyuubi.metadata.request.async.retry.enabled is `true`. | int | 1.6.0 |
| kyuubi.metadata.request.retry.interval | PT5S | The interval to check and trigger the metadata request retry tasks. | duration | 1.6.0 |
| kyuubi.metadata.search.window | <undefined> | The time window to search the metadata from metadata store. | duration | 1.10.1 |
| kyuubi.metadata.search.window | <undefined> | The time window to restrict user queries to metadata within a specific period. For example, if the window is set to P7D, only metadata from the past 7 days can be queried. If not set, it allows searching all metadata information in the metadata store. | duration | 1.10.1 |
| kyuubi.metadata.store.class | org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore | Fully qualified class name for server metadata store. | string | 1.6.0 |
| kyuubi.metadata.store.jdbc.database.schema.init | true | Whether to init the JDBC metadata store database schema. | boolean | 1.6.0 |
| kyuubi.metadata.store.jdbc.database.type | SQLITE | The database type for server jdbc metadata store.<ul> <li>SQLITE: SQLite3, JDBC driver `org.sqlite.JDBC`.</li> <li>MYSQL: MySQL, JDBC driver `com.mysql.cj.jdbc.Driver` (fallback `com.mysql.jdbc.Driver`).</li> <li>POSTGRESQL: PostgreSQL, JDBC driver `org.postgresql.Driver`.</li> <li>CUSTOM: User-defined database type, need to specify corresponding JDBC driver.</li> Note that: The JDBC datasource is powered by HiKariCP, for datasource properties, please specify them with the prefix: kyuubi.metadata.store.jdbc.datasource. For example, kyuubi.metadata.store.jdbc.datasource.connectionTimeout=10000. | string | 1.6.0 |
Expand Down
4 changes: 3 additions & 1 deletion kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2025,7 +2025,9 @@ object KyuubiConf {

val METADATA_SEARCH_WINDOW: OptionalConfigEntry[Long] =
buildConf("kyuubi.metadata.search.window")
.doc("The time window to search the metadata from metadata store.")
.doc("The time window to restrict user queries to metadata within a specific period. " +
"For example, if the window is set to P7D, only metadata from the past 7 days can be " +
"queried. If not set, it allows searching all metadata information in the metadata store.")
.version("1.10.1")
.timeConf
.checkValue(_ > 0, "must be positive number")
Expand Down
9 changes: 1 addition & 8 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,14 +431,7 @@ private[v1] class BatchesResource extends ApiRequestContext with Logging {
requestName = batchName,
createTime = createTimeFilter,
endTime = endTime)
val batches =
sessionManager.getBatchesFromMetadataStore(
filter,
from,
size,
desc,
// order by `create_time` rather then `key_id` if `create_time` is specified
orderByKeyId = createTimeFilter == 0)
val batches = sessionManager.getBatchesFromMetadataStore(filter, from, size, desc)
new GetBatchesResponse(from, batches.size, batches.asJava)
}

Expand Down
9 changes: 4 additions & 5 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataManager.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,15 +138,14 @@ class MetadataManager extends AbstractService("MetadataManager") {
filter: MetadataFilter,
from: Int,
size: Int,
desc: Boolean = false,
orderByKeyId: Boolean = true): Seq[Batch] = {
desc: Boolean = false): Seq[Batch] = {
withMetadataRequestMetrics(_metadataStore.getMetadataList(
filter,
from,
size,
desc,
orderByKeyId)).map(
buildBatch)
// for ascending order, select the metadata without order by, which is more efficient
orderBy = if (desc) Some("key_id") else None,
direction = if (desc) "DESC" else "ASC")).map(buildBatch)
}

def countBatch(
Expand Down
8 changes: 4 additions & 4 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataStore.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,16 @@ trait MetadataStore extends Closeable {
* @param from the metadata offset.
* @param size the size to get.
* @param desc the order of metadata list.
* @param orderByKeyId the result order by auto increment key_id, which is stable but might slow.
* If false, order by create_time.
* @param orderBy the order by column, default is the primary key, `key_id`.
* @param direction the order direction, default is `ASC`.
* @return selected metadata list.
*/
def getMetadataList(
filter: MetadataFilter,
from: Int,
size: Int,
desc: Boolean = false,
orderByKeyId: Boolean = true): Seq[Metadata]
orderBy: Option[String] = Some("key_id"),
direction: String = "ASC"): Seq[Metadata]

/**
* Count the metadata list with filter conditions.
Expand Down
7 changes: 3 additions & 4 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,16 +257,15 @@ class JDBCMetadataStore(conf: KyuubiConf) extends MetadataStore with Logging {
filter: MetadataFilter,
from: Int,
size: Int,
desc: Boolean = false,
orderByKeyId: Boolean = true): Seq[Metadata] = {
orderBy: Option[String] = Some("key_id"),
direction: String = "ASC"): Seq[Metadata] = {
val queryBuilder = new StringBuilder
val params = ListBuffer[Any]()
queryBuilder.append("SELECT ")
queryBuilder.append(METADATA_COLUMNS)
queryBuilder.append(s" FROM $METADATA_TABLE")
queryBuilder.append(s" ${assembleWhereClause(filter, params)}")
queryBuilder.append(" ORDER BY ").append(if (orderByKeyId) "key_id " else "create_time ")
queryBuilder.append(if (desc) "DESC " else "ASC ")
orderBy.foreach(o => queryBuilder.append(s" ORDER BY $o $direction"))
queryBuilder.append(dialect.limitClause(size, from))
val query = queryBuilder.toString
JdbcUtils.withConnection { connection =>
Expand Down
5 changes: 2 additions & 3 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,9 +280,8 @@ class KyuubiSessionManager private (name: String) extends SessionManager(name) {
filter: MetadataFilter,
from: Int,
size: Int,
desc: Boolean = false,
orderByKeyId: Boolean = true): Seq[Batch] = {
metadataManager.map(_.getBatches(filter, from, size, desc, orderByKeyId)).getOrElse(Seq.empty)
desc: Boolean = false): Seq[Batch] = {
metadataManager.map(_.getBatches(filter, from, size, desc)).getOrElse(Seq.empty)
}

def getBatchMetadata(batchId: String): Option[Metadata] = {
Expand Down

0 comments on commit 4568c77

Please sign in to comment.