From 4319763df018e68f1330d61ff1e04adb39893ce3 Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Wed, 18 Oct 2023 15:03:29 +0800 Subject: [PATCH 1/2] [KYUUBI #5457] [AUTHZ] Support RepairTable Commands for Hudi --- .../main/resources/table_command_spec.json | 14 ++++++++++ .../plugin/spark/authz/gen/HudiCommands.scala | 6 +++++ ...HudiCatalogRangerSparkExtensionSuite.scala | 26 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index abf4c314c0e..7269cd2a3e7 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -1585,6 +1585,20 @@ } ], "opType" : "DROPTABLE", "queryDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.RepairHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "tableName", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "MSCK", + "queryDescs" : [ ] }, { "classname" : "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand", "tableDescs" : [ { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala index a5f65c3d014..9cc78d039e2 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala @@ -116,6 +116,11 @@ object HudiCommands { DROPTABLE) } + val RepairHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.RepairHoodieTableCommand" + TableCommandSpec(cmd, Seq(TableDesc("tableName", classOf[TableIdentifierTableExtractor])), MSCK) + } + val TruncateHoodieTableCommand = { val cmd = "org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand" val columnDesc = ColumnDesc("partitionSpec", classOf[PartitionOptionColumnExtractor]) @@ -138,5 +143,6 @@ object HudiCommands { CreateHoodieTableAsSelectCommand, CreateHoodieTableLikeCommand, DropHoodieTableCommand, + RepairHoodieTableCommand, TruncateHoodieTableCommand) } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala index fc3ebf4fe74..8e89fe3841c 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala @@ -127,6 +127,7 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { interceptContains[AccessControlException]( doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 ADD COLUMNS(age int)")))( s"does not have [alter] privilege on [$namespace1/$table1]") + sql("set hoodie.schema.on.read.enable=false") } } @@ -240,6 +241,31 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { } } + test("RepairHoodieTableCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val repairTableSql = s"MSCK REPAIR TABLE $namespace1.$table1" + interceptContains[AccessControlException] { + doAs(someone, sql(repairTableSql)) + }(s"does not have [alter] privilege on [$namespace1/$table1]") + doAs(admin, sql(repairTableSql)) + } + } + test("TruncateHoodieTableCommand") { withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) From 85c209fa9d18f32a6d391e6ba2c06fd820ed0f6d Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Wed, 18 Oct 2023 16:58:18 +0800 Subject: [PATCH 2/2] Update HudiCatalogRangerSparkExtensionSuite.scala --- .../HudiCatalogRangerSparkExtensionSuite.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala index 8e89fe3841c..cc3c6738f5f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala @@ -123,11 +123,14 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { s" on [$namespace1/$table1]") // AlterTableCommand && Spark31AlterTableCommand - sql("set hoodie.schema.on.read.enable=true") - interceptContains[AccessControlException]( - doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 ADD COLUMNS(age int)")))( - s"does not have [alter] privilege on [$namespace1/$table1]") - sql("set hoodie.schema.on.read.enable=false") + try { + sql("set hoodie.schema.on.read.enable=true") + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 ADD COLUMNS(age int)")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + } finally { + sql("set hoodie.schema.on.read.enable=false") + } } }