[#1530] feat(spark-connector): support specifying scala version to bu…

…ild and test Gravitino (#1582)

### What changes were proposed in this pull request?
support specify scala versions, by two methods:
1. `gradle build -PscalaVersion=xx`
2. set `defaultScalaVersion` in gradle.properties

method2 will be overwrite  by method1 if both existing, 
and supports scala 2.12 for now.

### Why are the changes needed?
spark hive connector just supports scala 2.12

Fix: #1530 

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
existing UTs

build.gradle.kts

@@ -52,6 +52,11 @@ if (extra["jdkVersion"] !in listOf("8", "11", "17")) {
   )
 }
 
+val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
+if (scalaVersion !in listOf("2.12", "2.13")) {
+  throw GradleException("Found unsupported Scala version: $scalaVersion")
+}
+
 project.extra["extraJvmArgs"] = if (extra["jdkVersion"] in listOf("8", "11")) {
   listOf()
 } else {

gradle.properties

@@ -18,3 +18,6 @@ SONATYPE_PASSWORD = password
 #jdkVersion is used to specify the version of JDK to build and test Gravitino, current
 # supported version is 8, 11, and 17.
 jdkVersion = 8
+
+# defaultScalaVersion is used to specify the version of Scala to build and test Gravitino
+defaultScalaVersion = 2.12

gradle/libs.versions.toml

@@ -105,12 +105,8 @@ trino-testing= { group = "io.trino", name = "trino-testing", version.ref = "trin
 trino-memory= { group = "io.trino", name = "trino-memory", version.ref = "trino" }
 trino-cli= { group = "io.trino", name = "trino-cli", version.ref = "trino" }
 trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" }
-iceberg-spark-runtime = { group = "org.apache.iceberg", name = "iceberg-spark-runtime-3.4_2.13", version.ref = "iceberg" }
-spark-sql = { group = "org.apache.spark", name = "spark-sql_2.13", version.ref = "spark" }
-scala-collection-compat =  { group = "org.scala-lang.modules", name = "scala-collection-compat_2.13", version.ref = "scala-collection-compat" }
 sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" }
 testng = { group = "org.testng", name = "testng", version.ref = "testng" }
-spark-hive = { group = "org.apache.spark", name = "spark-hive_2.13", version.ref = "spark" }
 commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" }
 testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" }
 testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" }
@@ -153,4 +149,4 @@ shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow-plugin"
 node = { id = "com.github.node-gradle.node", version.ref = "node-plugin" }
 tasktree = {id = "com.dorongold.task-tree", version = "2.1.1"}
 dependencyLicenseReport = {id = "com.github.jk1.dependency-license-report", version = "2.5"}
-bom = {id = "org.cyclonedx.bom", version = "1.5.0"}
+bom = {id = "org.cyclonedx.bom", version = "1.5.0"}

integration-test/build.gradle.kts

@@ -12,6 +12,11 @@ plugins {
   id("idea")
 }
 
+val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
+val sparkVersion: String = libs.versions.spark.get()
+val icebergVersion: String = libs.versions.iceberg.get()
+val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()
+
 dependencies {
   implementation(project(":server"))
   implementation(project(":common"))
@@ -96,18 +101,18 @@ dependencies {
   testRuntimeOnly(libs.junit.jupiter.engine)
   testImplementation(libs.mockito.core)
   testImplementation(libs.bundles.log4j)
-  testImplementation(libs.iceberg.spark.runtime)
-  testImplementation(libs.spark.sql) {
+  testImplementation("org.apache.iceberg:iceberg-spark-runtime-3.4_$scalaVersion:$icebergVersion")
+  testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")
+  testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion")
+  testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
     exclude("org.apache.hadoop")
     exclude("org.rocksdb")
     exclude("org.apache.avro")
     exclude("org.apache.zookeeper")
     exclude("io.dropwizard.metrics")
   }
   testImplementation(libs.slf4j.jdk14)
-  testImplementation(libs.scala.collection.compat)
   testImplementation(libs.sqlite.jdbc)
-  testImplementation(libs.spark.hive)
   testImplementation(libs.testcontainers)
   testImplementation(libs.testcontainers.junit.jupiter)
   testImplementation(libs.testcontainers.mysql)