From bade42791417a90d91f672a90f3ebe4fcd8899f9 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 27 Nov 2023 14:54:51 +0800 Subject: [PATCH 1/8] [KYUUBI #5780][AUTHZ] Kyuubi tread PVM ass LeafNode to make logic more simple --- .../plugin/spark/authz/PrivilegesBuilder.scala | 9 +-------- .../authz/ranger/RangerSparkExtension.scala | 2 +- .../plugin/spark/authz/rule/Authorization.scala | 5 +---- .../rule/RuleEliminatePermanentViewMarker.scala | 17 +++++++++++++---- .../permanentview/PermanentViewMarker.scala | 9 ++------- .../RuleApplyPermanentViewMarker.scala | 9 ++------- .../ranger/RangerSparkExtensionSuite.scala | 2 +- 7 files changed, 21 insertions(+), 32 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala index d0f6e48ebe3..2d452ba9d67 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala @@ -28,7 +28,6 @@ import org.slf4j.LoggerFactory import org.apache.kyuubi.plugin.spark.authz.OperationType.OperationType import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType._ import org.apache.kyuubi.plugin.spark.authz.rule.Authorization._ -import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker import org.apache.kyuubi.plugin.spark.authz.rule.rowfilter._ import org.apache.kyuubi.plugin.spark.authz.serde._ import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ @@ -68,13 +67,7 @@ object PrivilegesBuilder { def mergeProjection(table: Table, plan: LogicalPlan): Unit = { if (projectionList.isEmpty) { - plan match { - case pvm: PermanentViewMarker - if pvm.isSubqueryExpressionPlaceHolder || pvm.output.isEmpty => - privilegeObjects += PrivilegeObject(table, pvm.outputColNames) - case _ => - privilegeObjects += PrivilegeObject(table, plan.output.map(_.name)) - } + privilegeObjects += PrivilegeObject(table, plan.output.map(_.name)) } else { val cols = (projectionList ++ conditionList).flatMap(collectLeaves) .filter(plan.outputSet.contains).map(_.name).distinct diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala index f6d43900370..93c10068a8c 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala @@ -53,7 +53,7 @@ class RangerSparkExtension extends (SparkSessionExtensions => Unit) { v1.injectResolutionRule(RuleApplyDataMaskingStage1) v1.injectOptimizerRule(_ => new RuleEliminateMarker()) v1.injectOptimizerRule(new RuleAuthorization(_)) - v1.injectOptimizerRule(_ => new RuleEliminatePermanentViewMarker()) + v1.injectOptimizerRule(new RuleEliminatePermanentViewMarker(_)) v1.injectOptimizerRule(_ => new RuleEliminateTypeOf()) v1.injectPlannerStrategy(new FilterDataSourceV2Strategy(_)) } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala index d83541825e4..eee8e004130 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala @@ -24,7 +24,6 @@ import org.apache.spark.sql.catalyst.trees.TreeNodeTag import org.apache.spark.sql.execution.SQLExecution.EXECUTION_ID_KEY import org.apache.kyuubi.plugin.spark.authz.rule.Authorization._ -import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker import org.apache.kyuubi.plugin.spark.authz.util.ReservedKeys._ abstract class Authorization(spark: SparkSession) extends Rule[LogicalPlan] { @@ -51,11 +50,9 @@ object Authorization { } } - protected def markAuthChecked(plan: LogicalPlan): LogicalPlan = { + def markAuthChecked(plan: LogicalPlan): LogicalPlan = { plan.setTagValue(KYUUBI_AUTHZ_TAG, ()) plan transformDown { - case pvm: PermanentViewMarker => - markAllNodesAuthChecked(pvm) case subquery: Subquery => markAllNodesAuthChecked(subquery) } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala index 864ada55f94..c1e0e5e20e1 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala @@ -20,18 +20,27 @@ package org.apache.kyuubi.plugin.spark.authz.rule import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule - import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker +import org.apache.spark.sql.SparkSession /** * Transforming up [[PermanentViewMarker]] */ -class RuleEliminatePermanentViewMarker extends Rule[LogicalPlan] { +class RuleEliminatePermanentViewMarker(sparkSession: SparkSession) extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { - plan.transformUp { - case pvm: PermanentViewMarker => pvm.child.transformAllExpressions { + var matched = false + val eliminatedPVM = plan.transformUp { + case pvm: PermanentViewMarker => + matched = true + pvm.child.transformAllExpressions { case s: SubqueryExpression => s.withNewPlan(apply(s.plan)) } } + if (matched) { + Authorization.markAuthChecked(eliminatedPVM) + sparkSession.sessionState.optimizer.execute(eliminatedPVM) + } else { + eliminatedPVM + } } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala index 18b58e4d8a3..11902bb8c51 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala @@ -19,20 +19,15 @@ package org.apache.kyuubi.plugin.spark.authz.rule.permanentview import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, LeafNode} import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild -case class PermanentViewMarker( - child: LogicalPlan, - catalogTable: CatalogTable, - outputColNames: Seq[String], - isSubqueryExpressionPlaceHolder: Boolean = false) extends UnaryNode +case class PermanentViewMarker(child: LogicalPlan, catalogTable: CatalogTable) extends LeafNode with WithInternalChild { override def output: Seq[Attribute] = child.output override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(child = newChild) - } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala index 1a0024abb7e..6a5cd51bad9 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala @@ -38,14 +38,9 @@ class RuleApplyPermanentViewMarker extends Rule[LogicalPlan] { case permanentView: View if hasResolvedPermanentView(permanentView) => val resolved = permanentView.transformAllExpressions { case subquery: SubqueryExpression => - subquery.withNewPlan(plan = - PermanentViewMarker( - subquery.plan, - permanentView.desc, - permanentView.output.map(_.name), - true)) + subquery.withNewPlan(plan = PermanentViewMarker(subquery.plan, permanentView.desc)) } - PermanentViewMarker(resolved, resolved.desc, resolved.output.map(_.name)) + PermanentViewMarker(resolved, resolved.desc) case other => apply(other) } } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala index 4cee0a1522b..c62cda5fae6 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala @@ -889,7 +889,7 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { sql(s"SELECT id as new_id, name, max_scope FROM $db1.$view1".stripMargin).show())) assert(e2.getMessage.contains( s"does not have [select] privilege on " + - s"[$db1/$view1/id,$db1/$view1/name,$db1/$view1/max_scope,$db1/$view1/sum_age]")) + s"[$db1/$view1/id,$db1/$view1/name,$db1/$view1/max_scope]")) } } } From 738d5062d30102f3e4227405317b57e1c03afc68 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 27 Nov 2023 15:17:44 +0800 Subject: [PATCH 2/8] Update RuleApplyPermanentViewMarker.scala --- .../rule/permanentview/RuleApplyPermanentViewMarker.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala index 6a5cd51bad9..87d7948af2f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala @@ -36,11 +36,11 @@ class RuleApplyPermanentViewMarker extends Rule[LogicalPlan] { plan mapChildren { case p: PermanentViewMarker => p case permanentView: View if hasResolvedPermanentView(permanentView) => - val resolved = permanentView.transformAllExpressions { + val resolved = permanentView.child.transformAllExpressions { case subquery: SubqueryExpression => subquery.withNewPlan(plan = PermanentViewMarker(subquery.plan, permanentView.desc)) } - PermanentViewMarker(resolved, resolved.desc) + PermanentViewMarker(resolved, permanentView.desc) case other => apply(other) } } From 57bf5ba3359c71da12309dc449e2d2b7c2f8f11c Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 27 Nov 2023 15:22:11 +0800 Subject: [PATCH 3/8] update --- .../spark/authz/rule/RuleEliminatePermanentViewMarker.scala | 3 ++- .../spark/authz/rule/permanentview/PermanentViewMarker.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala index c1e0e5e20e1..d468dcca614 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala @@ -17,11 +17,12 @@ package org.apache.kyuubi.plugin.spark.authz.rule +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule + import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker -import org.apache.spark.sql.SparkSession /** * Transforming up [[PermanentViewMarker]] diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala index 11902bb8c51..5ef8f4a748d 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.plugin.spark.authz.rule.permanentview import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, LeafNode} +import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild From 04a40c316f6da2bca68638dfbb9fed85aa54f9f9 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 27 Nov 2023 18:12:09 +0800 Subject: [PATCH 4/8] update --- .../authz/rule/permanentview/PermanentViewMarker.scala | 8 +------- .../plugin/spark/authz/util/WithInternalChildren.scala | 4 ---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala index 5ef8f4a748d..dd198c8b46a 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala @@ -21,13 +21,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} -import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild - -case class PermanentViewMarker(child: LogicalPlan, catalogTable: CatalogTable) extends LeafNode - with WithInternalChild { +case class PermanentViewMarker(child: LogicalPlan, catalogTable: CatalogTable) extends LeafNode { override def output: Seq[Attribute] = child.output - - override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = - copy(child = newChild) } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala index 582b34abee4..76598c3c0f0 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala @@ -24,10 +24,6 @@ trait WithInternalChildren { def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): LogicalPlan } -trait WithInternalChild { - def withNewChildInternal(newChild: LogicalPlan): LogicalPlan -} - trait WithInternalExpressionChild { def withNewChildInternal(newChild: Expression): Expression } From 5dbe232fc3b187854cc13743b09d850d6420f359 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Mon, 27 Nov 2023 18:13:16 +0800 Subject: [PATCH 5/8] Update WithInternalChildren.scala --- .../kyuubi/plugin/spark/authz/util/WithInternalChildren.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala index 76598c3c0f0..582b34abee4 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/WithInternalChildren.scala @@ -24,6 +24,10 @@ trait WithInternalChildren { def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): LogicalPlan } +trait WithInternalChild { + def withNewChildInternal(newChild: LogicalPlan): LogicalPlan +} + trait WithInternalExpressionChild { def withNewChildInternal(newChild: Expression): Expression } From 2113cf51b703e47d40277a81898f12a416af94c9 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Tue, 28 Nov 2023 16:44:36 +0800 Subject: [PATCH 6/8] Update RuleApplyPermanentViewMarker.scala --- .../authz/rule/permanentview/RuleApplyPermanentViewMarker.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala index 87d7948af2f..b809d8f34ef 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala @@ -36,7 +36,7 @@ class RuleApplyPermanentViewMarker extends Rule[LogicalPlan] { plan mapChildren { case p: PermanentViewMarker => p case permanentView: View if hasResolvedPermanentView(permanentView) => - val resolved = permanentView.child.transformAllExpressions { + val resolved = permanentView.transformAllExpressions { case subquery: SubqueryExpression => subquery.withNewPlan(plan = PermanentViewMarker(subquery.plan, permanentView.desc)) } From fbc989a7ebc1a84e383081c7f021213ae2bba0c8 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Tue, 28 Nov 2023 17:13:12 +0800 Subject: [PATCH 7/8] Update Authorization.scala --- .../plugin/spark/authz/rule/Authorization.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala index eee8e004130..d1494266e85 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala @@ -18,7 +18,7 @@ package org.apache.kyuubi.plugin.spark.authz.rule import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery} +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreeNodeTag import org.apache.spark.sql.execution.SQLExecution.EXECUTION_ID_KEY @@ -53,16 +53,15 @@ object Authorization { def markAuthChecked(plan: LogicalPlan): LogicalPlan = { plan.setTagValue(KYUUBI_AUTHZ_TAG, ()) plan transformDown { - case subquery: Subquery => - markAllNodesAuthChecked(subquery) + // TODO: Add this line Support for spark3.1, we can remove this + // after spark 3.2 since https://issues.apache.org/jira/browse/SPARK-34269 + case view: View => + markAllNodesAuthChecked(view.child) } } protected def isAuthChecked(plan: LogicalPlan): Boolean = { - plan match { - case subquery: Subquery => isAuthChecked(subquery.child) - case p => p.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty - } + plan.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty } def setExplainCommandExecutionId(sparkSession: SparkSession): Unit = { From 64f7947c1fe47c44e3989c0a016d0b6d1a0b0242 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Tue, 28 Nov 2023 18:39:36 +0800 Subject: [PATCH 8/8] update --- .../main/resources/database_command_spec.json | 2 +- .../main/resources/function_command_spec.json | 2 +- .../src/main/resources/scan_command_spec.json | 2 +- .../main/resources/table_command_spec.json | 23 ++++++++++++------- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json index 3918186ac80..5891fb1e548 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json @@ -215,4 +215,4 @@ } ], "opType" : "SWITCHDATABASE", "uriDescs" : [ ] -} ] +} ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json index 8644f9860ca..14dad8e2a3f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json @@ -111,4 +111,4 @@ "comment" : "" } ], "opType" : "RELOADFUNCTION" -} ] +} ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json index ba4d790e84c..1145adbe07a 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json @@ -111,4 +111,4 @@ "comment" : "" } ], "uriDescs" : [ ] -} ] +} ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index 27b205309cf..b555bbcf8be 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -16,7 +16,8 @@ "uriDescs" : [ { "fieldName" : "child", "fieldExtractor" : "ResolvedTableURIExtractor", - "isInput" : false + "isInput" : false, + "comment" : "" } ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AddPartitions", @@ -52,7 +53,8 @@ "uriDescs" : [ { "fieldName" : "child", "fieldExtractor" : "ResolvedTableURIExtractor", - "isInput" : false + "isInput" : false, + "comment" : "" } ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AlterTable", @@ -72,7 +74,8 @@ "uriDescs" : [ { "fieldName" : "ident", "fieldExtractor" : "IdentifierURIExtractor", - "isInput" : false + "isInput" : false, + "comment" : "" } ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AppendData", @@ -334,7 +337,8 @@ "uriDescs" : [ { "fieldName" : "child", "fieldExtractor" : "ResolvedTableURIExtractor", - "isInput" : false + "isInput" : false, + "comment" : "" } ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropPartitions", @@ -497,7 +501,8 @@ "uriDescs" : [ { "fieldName" : "child", "fieldExtractor" : "ResolvedTableURIExtractor", - "isInput" : false + "isInput" : false, + "comment" : "" } ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RenamePartitions", @@ -549,7 +554,8 @@ "uriDescs" : [ { "fieldName" : "child", "fieldExtractor" : "ResolvedTableURIExtractor", - "isInput" : false + "isInput" : false, + "comment" : "" } ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceData", @@ -703,7 +709,8 @@ "uriDescs" : [ { "fieldName" : "table", "fieldExtractor" : "ResolvedTableURIExtractor", - "isInput" : false + "isInput" : false, + "comment" : "" } ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable", @@ -2556,4 +2563,4 @@ "isInput" : false, "comment" : "Delta" } ] -} ] +} ] \ No newline at end of file