From 27ce25b8a438edef59d6182888f12aa695f082fb Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 22 Jan 2024 14:13:49 +0800 Subject: [PATCH] [#1530] feat(spark-connector): support specifying scala version to build and test Gravitino (#1582) ### What changes were proposed in this pull request? support specify scala versions, by two methods: 1. `gradle build -PscalaVersion=xx` 2. set `defaultScalaVersion` in gradle.properties method2 will be overwrite by method1 if both existing, and supports scala 2.12 for now. ### Why are the changes needed? spark hive connector just supports scala 2.12 Fix: #1530 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing UTs --- build.gradle.kts | 5 +++++ gradle.properties | 3 +++ gradle/libs.versions.toml | 6 +----- integration-test/build.gradle.kts | 13 +++++++++---- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b585bddc6df..be67c44042c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -52,6 +52,11 @@ if (extra["jdkVersion"] !in listOf("8", "11", "17")) { ) } +val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() +if (scalaVersion !in listOf("2.12", "2.13")) { + throw GradleException("Found unsupported Scala version: $scalaVersion") +} + project.extra["extraJvmArgs"] = if (extra["jdkVersion"] in listOf("8", "11")) { listOf() } else { diff --git a/gradle.properties b/gradle.properties index a342ae14d79..6bf51d2cddb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -18,3 +18,6 @@ SONATYPE_PASSWORD = password #jdkVersion is used to specify the version of JDK to build and test Gravitino, current # supported version is 8, 11, and 17. jdkVersion = 8 + +# defaultScalaVersion is used to specify the version of Scala to build and test Gravitino +defaultScalaVersion = 2.12 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bba5cf7dd3a..78a9a5f1318 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -105,12 +105,8 @@ trino-testing= { group = "io.trino", name = "trino-testing", version.ref = "trin trino-memory= { group = "io.trino", name = "trino-memory", version.ref = "trino" } trino-cli= { group = "io.trino", name = "trino-cli", version.ref = "trino" } trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" } -iceberg-spark-runtime = { group = "org.apache.iceberg", name = "iceberg-spark-runtime-3.4_2.13", version.ref = "iceberg" } -spark-sql = { group = "org.apache.spark", name = "spark-sql_2.13", version.ref = "spark" } -scala-collection-compat = { group = "org.scala-lang.modules", name = "scala-collection-compat_2.13", version.ref = "scala-collection-compat" } sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" } testng = { group = "org.testng", name = "testng", version.ref = "testng" } -spark-hive = { group = "org.apache.spark", name = "spark-hive_2.13", version.ref = "spark" } commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" } testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" } testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" } @@ -153,4 +149,4 @@ shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow-plugin" node = { id = "com.github.node-gradle.node", version.ref = "node-plugin" } tasktree = {id = "com.dorongold.task-tree", version = "2.1.1"} dependencyLicenseReport = {id = "com.github.jk1.dependency-license-report", version = "2.5"} -bom = {id = "org.cyclonedx.bom", version = "1.5.0"} \ No newline at end of file +bom = {id = "org.cyclonedx.bom", version = "1.5.0"} diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index ac5e7af52f9..bc3c7470c51 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -12,6 +12,11 @@ plugins { id("idea") } +val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() +val sparkVersion: String = libs.versions.spark.get() +val icebergVersion: String = libs.versions.iceberg.get() +val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get() + dependencies { implementation(project(":server")) implementation(project(":common")) @@ -96,8 +101,10 @@ dependencies { testRuntimeOnly(libs.junit.jupiter.engine) testImplementation(libs.mockito.core) testImplementation(libs.bundles.log4j) - testImplementation(libs.iceberg.spark.runtime) - testImplementation(libs.spark.sql) { + testImplementation("org.apache.iceberg:iceberg-spark-runtime-3.4_$scalaVersion:$icebergVersion") + testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") + testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion") + testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { exclude("org.apache.hadoop") exclude("org.rocksdb") exclude("org.apache.avro") @@ -105,9 +112,7 @@ dependencies { exclude("io.dropwizard.metrics") } testImplementation(libs.slf4j.jdk14) - testImplementation(libs.scala.collection.compat) testImplementation(libs.sqlite.jdbc) - testImplementation(libs.spark.hive) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql)