From 4d54796b4471eac56abf6374afdb6f2f2260ba55 Mon Sep 17 00:00:00 2001 From: fanng Date: Wed, 17 Jan 2024 20:32:30 +0800 Subject: [PATCH 1/4] xx --- gradle.properties | 6 ++++++ gradle/libs.versions.toml | 6 +----- integration-test/build.gradle.kts | 15 +++++++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/gradle.properties b/gradle.properties index a342ae14d79..0492cdfef27 100644 --- a/gradle.properties +++ b/gradle.properties @@ -18,3 +18,9 @@ SONATYPE_PASSWORD = password #jdkVersion is used to specify the version of JDK to build and test Gravitino, current # supported version is 8, 11, and 17. jdkVersion = 8 + +# defaultScalaVersion is used to specify the version of Scala to build and test Gravitino +defaultScalaVersion = 2.12 + +# defaultSparkVersion is used to specify the version of Spark to build and test Gravitino +defaultSparkVersion = 3.4 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bba5cf7dd3a..78a9a5f1318 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -105,12 +105,8 @@ trino-testing= { group = "io.trino", name = "trino-testing", version.ref = "trin trino-memory= { group = "io.trino", name = "trino-memory", version.ref = "trino" } trino-cli= { group = "io.trino", name = "trino-cli", version.ref = "trino" } trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" } -iceberg-spark-runtime = { group = "org.apache.iceberg", name = "iceberg-spark-runtime-3.4_2.13", version.ref = "iceberg" } -spark-sql = { group = "org.apache.spark", name = "spark-sql_2.13", version.ref = "spark" } -scala-collection-compat = { group = "org.scala-lang.modules", name = "scala-collection-compat_2.13", version.ref = "scala-collection-compat" } sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" } testng = { group = "org.testng", name = "testng", version.ref = "testng" } -spark-hive = { group = "org.apache.spark", name = "spark-hive_2.13", version.ref = "spark" } commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" } testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" } testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" } @@ -153,4 +149,4 @@ shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow-plugin" node = { id = "com.github.node-gradle.node", version.ref = "node-plugin" } tasktree = {id = "com.dorongold.task-tree", version = "2.1.1"} dependencyLicenseReport = {id = "com.github.jk1.dependency-license-report", version = "2.5"} -bom = {id = "org.cyclonedx.bom", version = "1.5.0"} \ No newline at end of file +bom = {id = "org.cyclonedx.bom", version = "1.5.0"} diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index ac5e7af52f9..1e9e217ac62 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -12,6 +12,12 @@ plugins { id("idea") } +val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() +val sparkMajorVersion: String = project.properties["sparkVersion"] as? String ?: extra["defaultSparkVersion"].toString() +val sparkVersion: String = libs.versions.spark.get() +val icebergVersion: String = libs.versions.iceberg.get() +val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get() + dependencies { implementation(project(":server")) implementation(project(":common")) @@ -96,8 +102,10 @@ dependencies { testRuntimeOnly(libs.junit.jupiter.engine) testImplementation(libs.mockito.core) testImplementation(libs.bundles.log4j) - testImplementation(libs.iceberg.spark.runtime) - testImplementation(libs.spark.sql) { + testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") + testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") + testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion") + testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { exclude("org.apache.hadoop") exclude("org.rocksdb") exclude("org.apache.avro") @@ -105,9 +113,8 @@ dependencies { exclude("io.dropwizard.metrics") } testImplementation(libs.slf4j.jdk14) - testImplementation(libs.scala.collection.compat) testImplementation(libs.sqlite.jdbc) - testImplementation(libs.spark.hive) + testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) From 272538f800a2d88f9f7a725b028f04e65059379c Mon Sep 17 00:00:00 2001 From: fanng Date: Mon, 22 Jan 2024 11:33:31 +0800 Subject: [PATCH 2/4] remove spark version --- build.gradle.kts | 5 +++++ gradle.properties | 3 --- integration-test/build.gradle.kts | 3 +-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b585bddc6df..c6b4de7893d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -52,6 +52,11 @@ if (extra["jdkVersion"] !in listOf("8", "11", "17")) { ) } +val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() +if (scalaVersion !in listOf("2.12")) { + throw GradleException("Found unsupported Scala version: $scalaVersion") +} + project.extra["extraJvmArgs"] = if (extra["jdkVersion"] in listOf("8", "11")) { listOf() } else { diff --git a/gradle.properties b/gradle.properties index 0492cdfef27..6bf51d2cddb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,6 +21,3 @@ jdkVersion = 8 # defaultScalaVersion is used to specify the version of Scala to build and test Gravitino defaultScalaVersion = 2.12 - -# defaultSparkVersion is used to specify the version of Spark to build and test Gravitino -defaultSparkVersion = 3.4 diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 1e9e217ac62..86fed3061c0 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -13,7 +13,6 @@ plugins { } val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() -val sparkMajorVersion: String = project.properties["sparkVersion"] as? String ?: extra["defaultSparkVersion"].toString() val sparkVersion: String = libs.versions.spark.get() val icebergVersion: String = libs.versions.iceberg.get() val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get() @@ -102,7 +101,7 @@ dependencies { testRuntimeOnly(libs.junit.jupiter.engine) testImplementation(libs.mockito.core) testImplementation(libs.bundles.log4j) - testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") + testImplementation("org.apache.iceberg:iceberg-spark-runtime-3.4_$scalaVersion:$icebergVersion") testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion") testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { From 277b65625ba23d6190ddc3f96cd8d12663d41fb0 Mon Sep 17 00:00:00 2001 From: fanng Date: Mon, 22 Jan 2024 11:43:39 +0800 Subject: [PATCH 3/4] remove empty line --- integration-test/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 86fed3061c0..bc3c7470c51 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -113,7 +113,6 @@ dependencies { } testImplementation(libs.slf4j.jdk14) testImplementation(libs.sqlite.jdbc) - testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) From ed004e503bd56c0cb7a0b64acfec35b711f10c37 Mon Sep 17 00:00:00 2001 From: fanng Date: Mon, 22 Jan 2024 12:42:56 +0800 Subject: [PATCH 4/4] add 2.13 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index c6b4de7893d..be67c44042c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -53,7 +53,7 @@ if (extra["jdkVersion"] !in listOf("8", "11", "17")) { } val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString() -if (scalaVersion !in listOf("2.12")) { +if (scalaVersion !in listOf("2.12", "2.13")) { throw GradleException("Found unsupported Scala version: $scalaVersion") }