[apache#4759] feat(CI): Use Spark verify Ranger authorization Hive

xunliu · Sep 18, 2024 · 905f8f9 · 905f8f9
1 parent 3433967
commit 905f8f9
Show file tree

Hide file tree

Showing 14 changed files with 461 additions and 316 deletions.
diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts
@@ -24,6 +24,10 @@ plugins {
   id("idea")
 }
 
+val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
+val sparkVersion: String = libs.versions.spark35.get()
+val kyuubiVersion: String = libs.versions.kyuubi4spark35.get()
+
 dependencies {
   implementation(project(":api")) {
     exclude(group = "*")
@@ -62,7 +66,6 @@ dependencies {
 
   testImplementation(project(":common"))
   testImplementation(project(":clients:client-java"))
-  testImplementation(project(":server"))
   testImplementation(project(":catalogs:catalog-common"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(libs.junit.jupiter.api)
@@ -80,13 +83,24 @@ dependencies {
     exclude("org.elasticsearch.plugin")
     exclude("javax.ws.rs")
     exclude("org.apache.ranger", "ranger-plugin-classloader")
+    exclude("com.amazonaws", "aws-java-sdk-bundle")
+    exclude("org.eclipse.jetty")
+    exclude("org.apache.hadoop")
   }
-  testImplementation(libs.hive2.jdbc) {
-    exclude("org.slf4j")
-    exclude("org.eclipse.jetty.aggregate")
-  }
+  testImplementation(libs.h2db)
   testImplementation(libs.mysql.driver)
   testImplementation(libs.postgresql.driver)
+  testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
+    exclude("org.apache.avro")
+    exclude("org.apache.hadoop")
+    exclude("org.apache.zookeeper")
+    exclude("io.dropwizard.metrics")
+    exclude("org.rocksdb")
+  }
+  testImplementation("org.apache.kyuubi:kyuubi-spark-authz_$scalaVersion:$kyuubiVersion") {
+    exclude("com.sun.jersey")
+  }
 }
 
 tasks {
@@ -96,7 +110,7 @@ tasks {
   }
 
   val copyAuthorizationLibs by registering(Copy::class) {
-    dependsOn("jar", "runtimeJars")
+    dependsOn("jar", runtimeJars)
     from("build/libs") {
       exclude("guava-*.jar")
       exclude("log4j-*.jar")
@@ -120,4 +134,7 @@ tasks.test {
   } else {
     dependsOn(tasks.jar)
   }
+  doFirst {
+    environment("HADOOP_USER_NAME", "gravitino")
+  }
 }