[apache#4759] feat(CI): Use Spark verify Ranger authorization Hive

xunliu · Sep 16, 2024 · 95a80e0 · 95a80e0
1 parent 4f961bb
commit 95a80e0
Show file tree

Hide file tree

Showing 12 changed files with 450 additions and 292 deletions.
diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts
@@ -24,6 +24,9 @@ plugins {
   id("idea")
 }
 
+val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
+val sparkVersion: String = libs.versions.spark35.get()
+
 dependencies {
   implementation(project(":api")) {
     exclude(group = "*")
@@ -58,11 +61,9 @@ dependencies {
     exclude("javax.ws.rs")
     exclude("org.eclipse.jetty")
   }
-  implementation(libs.rome)
 
   testImplementation(project(":common"))
   testImplementation(project(":clients:client-java"))
-  testImplementation(project(":server"))
   testImplementation(project(":catalogs:catalog-common"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(libs.junit.jupiter.api)
@@ -80,13 +81,24 @@ dependencies {
     exclude("org.elasticsearch.plugin")
     exclude("javax.ws.rs")
     exclude("org.apache.ranger", "ranger-plugin-classloader")
+    exclude("com.amazonaws", "aws-java-sdk-bundle")
+    exclude("org.eclipse.jetty")
+    exclude("org.apache.hadoop")
   }
-  testImplementation(libs.hive2.jdbc) {
-    exclude("org.slf4j")
-    exclude("org.eclipse.jetty.aggregate")
-  }
+  testImplementation(libs.h2db)
   testImplementation(libs.mysql.driver)
   testImplementation(libs.postgresql.driver)
+  testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
+    exclude("org.apache.avro")
+    exclude("org.apache.hadoop")
+    exclude("org.apache.zookeeper")
+    exclude("io.dropwizard.metrics")
+    exclude("org.rocksdb")
+  }
+  testImplementation("org.apache.kyuubi:kyuubi-spark-authz_2.12:1.9.2") {
+    exclude("com.sun.jersey")
+  }
 }
 
 tasks {
@@ -96,7 +108,7 @@ tasks {
   }
 
   val copyAuthorizationLibs by registering(Copy::class) {
-    dependsOn("jar", "runtimeJars")
+    dependsOn("jar", runtimeJars)
     from("build/libs") {
       exclude("guava-*.jar")
       exclude("log4j-*.jar")
@@ -120,4 +132,8 @@ tasks.test {
   } else {
     dependsOn(tasks.jar)
   }
+  doFirst {
+    environment("HADOOP_USER_NAME", "gravitino")
+    environment("GRAVITINO_HOME", rootDir.path)
+  }
 }