Merge branch 'master' into add_option_comment_for_authz

apache · Nov 23, 2023 · a125ee4 · a125ee4
2 parents 80eb9ff + fdcb456
commit a125ee4
Show file tree

Hide file tree

Showing 12 changed files with 326 additions and 84 deletions.
diff --git a/charts/kyuubi/templates/kyuubi-service.yaml b/charts/kyuubi/templates/kyuubi-service.yaml
@@ -37,6 +37,12 @@ spec:
       {{- end }}
   selector:
     {{- include "kyuubi.selectorLabels" $ | nindent 4 }}
+  {{- if ($frontend.service.sessionAffinity) }}
+  sessionAffinity: {{ $frontend.service.sessionAffinity }}
+  {{- end }}
+  {{- with $frontend.service.sessionAffinityConfig }}
+  sessionAffinityConfig: {{- toYaml . | nindent 4 }}
+  {{- end }}
 ---
 {{- end }}
 {{- end }}
diff --git a/charts/kyuubi/values.yaml b/charts/kyuubi/values.yaml
@@ -85,6 +85,13 @@ server:
       port: "{{ .Values.server.thriftBinary.port }}"
       nodePort: ~
       annotations: {}
+      # candidates are ClientIP or None
+      # https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/
+      sessionAffinity: ~
+      sessionAffinityConfig: {}
+      #  sessionAffinityConfig:
+      #    clientIP:
+      #      timeoutSeconds: 10800
 
   # Thrift HTTP protocol (HiveServer2 compatible)
   thriftHttp:
@@ -95,6 +102,13 @@ server:
       port: "{{ .Values.server.thriftHttp.port }}"
       nodePort: ~
       annotations: {}
+      # candidates are ClientIP or None
+      # https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/
+      sessionAffinity: ~
+      sessionAffinityConfig: {}
+      #  sessionAffinityConfig:
+      #    clientIP:
+      #      timeoutSeconds: 10800
 
   # REST API protocol (experimental)
   rest:
@@ -105,6 +119,13 @@ server:
       port: "{{ .Values.server.rest.port }}"
       nodePort: ~
       annotations: {}
+      # candidates are ClientIP or None
+      # https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/
+      sessionAffinity: ~
+      sessionAffinityConfig: {}
+      #  sessionAffinityConfig:
+      #    clientIP:
+      #      timeoutSeconds: 10800
 
   # MySQL compatible text protocol (experimental)
   mysql:
@@ -115,6 +136,13 @@ server:
       port: "{{ .Values.server.mysql.port }}"
       nodePort: ~
       annotations: {}
+      # candidates are ClientIP or None
+      # https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/
+      sessionAffinity: ~
+      sessionAffinityConfig: {}
+      #  sessionAffinityConfig:
+      #    clientIP:
+      #      timeoutSeconds: 10800
 
 monitoring:
   # Exposes metrics in Prometheus format

diff --git a/conf/kyuubi-env.sh.template b/conf/kyuubi-env.sh.template
@@ -64,5 +64,5 @@
 # export HIVE_HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/common/lib/commons-collections-3.2.2.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-runtime-3.1.0.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-api-3.1.0.jar:${HADOOP_HOME}/share/hadoop/common/lib/htrace-core4-4.1.0-incubating.jar
 # export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf
 # export YARN_CONF_DIR=/usr/ndp/current/yarn/conf
-# export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m  -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m"
-# export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark"
+# export KYUUBI_JAVA_OPTS="-Xmx10g -XX:MaxMetaspaceSize=512m -XX:MaxDirectMemorySize=1024m -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+UnlockDiagnosticVMOptions -XX:+UseCondCardMark -XX:+UseGCOverheadLimit -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -verbose:gc -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"
+# export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UseCondCardMark"
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -2389,7 +2389,19 @@
   } ],
   "opType" : "MSCK",
   "queryDescs" : [ ],
-  "uriDescs" : [ ]
+  "uriDescs" : [ {
+    "fieldName" : "child",
+    "fieldExtractor" : "ResolvedTableURIExtractor",
+    "isInput" : false
+  }, {
+    "fieldName" : "table",
+    "fieldExtractor" : "TableIdentifierOptionURIExtractor",
+    "isInput" : false
+  }, {
+    "fieldName" : "path",
+    "fieldExtractor" : "StringURIExtractor",
+    "isInput" : false
+  } ]
 }, {
   "classname" : "org.apache.spark.sql.delta.commands.DeleteCommand",
   "tableDescs" : [ {

diff --git a/...i-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala b/...i-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
@@ -65,7 +65,11 @@ object DeltaCommands extends CommandSpecs[TableCommandSpec] {
     val childDesc = TableDesc("child", classOf[ResolvedTableTableExtractor], comment = "Delta")
     val tableDesc =
       TableDesc("table", classOf[TableIdentifierOptionTableExtractor], comment = "Delta")
-    TableCommandSpec(cmd, Seq(childDesc, tableDesc), MSCK)
+    val uriDescs = Seq(
+      UriDesc("child", classOf[ResolvedTableURIExtractor]),
+      UriDesc("table", classOf[TableIdentifierOptionURIExtractor]),
+      UriDesc("path", classOf[StringURIExtractor]))
+    TableCommandSpec(cmd, Seq(childDesc, tableDesc), MSCK, uriDescs = uriDescs)
   }
 
   override def specs: Seq[TableCommandSpec] = Seq(

diff --git a/...a/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala b/...a/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala
@@ -451,6 +451,23 @@ class DeltaCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
       doAs(admin, sql(optimizeTableSql2))
     })
   }
+
+  test("vacuum path-based table") {
+    withTempDir(path => {
+      doAs(admin, sql(createPathBasedTableSql(path)))
+      val vacuumTableSql1 = s"VACUUM delta.`$path`"
+      interceptEndsWith[AccessControlException](
+        doAs(someone, sql(vacuumTableSql1)))(
+        s"does not have [write] privilege on [[$path, $path/]]")
+      doAs(admin, sql(vacuumTableSql1))
+
+      val vacuumTableSql2 = s"VACUUM '$path'"
+      interceptEndsWith[AccessControlException](
+        doAs(someone, sql(vacuumTableSql2)))(
+        s"does not have [write] privilege on [[$path, $path/]]")
+      doAs(admin, sql(vacuumTableSql2))
+    })
+  }
 }
 
 object DeltaCatalogRangerSparkExtensionSuite {