diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index cc88ebe9fa0..0071eff3369 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -32,26 +32,25 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat dependencies { implementation(project(":api")) { - exclude("*") + exclude("*") } implementation(project(":catalogs:catalog-common")) implementation(project(":common")) { - exclude("*") + exclude("*") } implementation(project(":core")) { - exclude("*") + exclude("*") } implementation(project(":iceberg:iceberg-common")) implementation(libs.bundles.iceberg) - implementation(libs.bundles.jersey) - implementation(libs.bundles.jetty) + implementation(libs.bundles.log4j) implementation(libs.cglib) implementation(libs.commons.collections4) implementation(libs.commons.io) implementation(libs.commons.lang3) implementation(libs.guava) - implementation(libs.sqlite.jdbc) + // implementation(libs.sqlite.jdbc) annotationProcessor(libs.lombok) @@ -65,7 +64,10 @@ dependencies { testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion") testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") - testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") + testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") { + // exclude("org.apache.hadoop") + exclude("org.apache.hive") + } testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { exclude("org.apache.avro") exclude("org.apache.hadoop") @@ -74,22 +76,15 @@ dependencies { exclude("org.rocksdb") } - testImplementation(libs.hadoop2.common) { - exclude("com.github.spotbugs") - } - testImplementation(libs.jersey.test.framework.core) { - exclude(group = "org.junit.jupiter") - } - testImplementation(libs.jersey.test.framework.provider.jetty) { - exclude(group = "org.junit.jupiter") - } + testImplementation(libs.bundles.jersey) + testImplementation(libs.bundles.jetty) + // testImplementation(libs.commons.lang3) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mockito.core) // For test TestMultipleJDBCLoad, it was depended on testcontainers.mysql and testcontainers.postgresql testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) - testImplementation(libs.slf4j.api) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) @@ -110,7 +105,11 @@ tasks { val copyCatalogLibs by registering(Copy::class) { dependsOn("jar", "runtimeJars") - from("build/libs") + from("build/libs") { + exclude("guava-*.jar") + exclude("log4j-*.jar") + exclude("slf4j-*.jar") + } into("$rootDir/distribution/package/catalogs/lakehouse-iceberg/libs") } @@ -158,6 +157,11 @@ tasks.test { } } +tasks.register("copy") { + from(configurations.testRuntimeClasspath) + into("build/libs-runtime") +} + tasks.clean { delete("spark-warehouse") } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java index 04f0aac8ce1..004e73aac12 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java @@ -26,7 +26,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; -import org.apache.commons.lang.math.RandomUtils; +import java.util.Random; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergColumn; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.expressions.Expression; @@ -54,6 +54,8 @@ public class TestBaseConvert { protected static final Map GRAVITINO_TYPE = new HashMap<>(); protected static final Map ICEBERG_TYPE = new HashMap<>(); + private static Random random = new Random(System.currentTimeMillis()); + static { GRAVITINO_TYPE.put("BOOLEAN", org.apache.gravitino.rel.types.Types.BooleanType.get()); // Types not supported by iceberg @@ -113,8 +115,8 @@ protected static SortOrder[] createSortOrder(String... colNames) { results.add( SortOrders.of( field(colName), - RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, - RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST)); + random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, + random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST)); } return results.toArray(new SortOrder[0]); } @@ -122,15 +124,15 @@ protected static SortOrder[] createSortOrder(String... colNames) { protected static SortOrder createSortOrder(String name, String colName) { return SortOrders.of( FunctionExpression.of(name, field(colName)), - RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, - RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); + random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, + random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); } protected static SortOrder createSortOrder(String name, int width, String colName) { return SortOrders.of( FunctionExpression.of(name, Literals.integerLiteral(width), field(colName)), - RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, - RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); + random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, + random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); } protected static Types.NestedField createNestedField( @@ -143,7 +145,7 @@ protected static Types.NestedField[] createNestedField(String... colNames) { for (int i = 0; i < colNames.length; i++) { results.add( Types.NestedField.of( - i + 1, RandomUtils.nextBoolean(), colNames[i], getRandomIcebergType(), TEST_COMMENT)); + i + 1, random.nextBoolean(), colNames[i], getRandomIcebergType(), TEST_COMMENT)); } return results.toArray(new Types.NestedField[0]); } @@ -211,7 +213,7 @@ protected static String getGravitinoSortOrderExpressionString(Expression sortOrd private static Type getRandomGravitinoType() { Collection values = GRAVITINO_TYPE.values(); return values.stream() - .skip(RandomUtils.nextInt(values.size())) + .skip(random.nextInt(values.size())) .findFirst() .orElseThrow(() -> new RuntimeException("No type found")); } @@ -219,7 +221,7 @@ private static Type getRandomGravitinoType() { private static org.apache.iceberg.types.Type getRandomIcebergType() { Collection values = ICEBERG_TYPE.values(); return values.stream() - .skip(RandomUtils.nextInt(values.size())) + .skip(random.nextInt(values.size())) .findFirst() .orElseThrow(() -> new RuntimeException("No type found")); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 968190c483d..bed6bbd13d8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -137,6 +137,7 @@ hive2-common = { group = "org.apache.hive", name = "hive-common", version.ref = hive2-jdbc = { group = "org.apache.hive", name = "hive-jdbc", version.ref = "hive2"} hadoop2-auth = { group = "org.apache.hadoop", name = "hadoop-auth", version.ref = "hadoop2" } hadoop2-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop2" } +hadoop2-hdfs-client = { group = "org.apache.hadoop", name = "hadoop-hdfs-client", version.ref = "hadoop2" } hadoop2-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop2"} hadoop2-mapreduce-client-core = { group = "org.apache.hadoop", name = "hadoop-mapreduce-client-core", version.ref = "hadoop2"} hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop3" } diff --git a/iceberg/iceberg-common/build.gradle.kts b/iceberg/iceberg-common/build.gradle.kts index bb67907a622..c9f72b905a4 100644 --- a/iceberg/iceberg-common/build.gradle.kts +++ b/iceberg/iceberg-common/build.gradle.kts @@ -51,7 +51,7 @@ dependencies { exclude("org.apache.zookeeper") exclude("org.mortbay.jetty") } - implementation(libs.hadoop2.hdfs) { + implementation(libs.hadoop2.hdfs.client) { exclude("com.sun.jersey") exclude("javax.servlet") exclude("org.fusesource.leveldbjni") @@ -60,9 +60,6 @@ dependencies { implementation(libs.hadoop2.mapreduce.client.core) { // exclude("*") exclude("*") - exclude("com.sun.jersey") - exclude("javax.servlet") - exclude("org.mortbay.jetty") } implementation(libs.hive2.metastore) { exclude("co.cask.tephra") @@ -71,23 +68,15 @@ dependencies { exclude("com.sun.jersey") exclude("com.tdunning", "json") exclude("com.zaxxer", "HikariCP") - // exclude("com.github.joshelser") - // exclude("io.dropwizard.metrics") exclude("com.github.joshelser") exclude("io.dropwizard.metrics") exclude("javax.servlet") exclude("javax.transaction", "transaction-api") - // exclude("org.apache.ant") exclude("org.apache.ant") exclude("org.apache.avro", "avro") - // exclude("org.apache.curator") - // exclude("org.apache.derby") exclude("org.apache.curator") exclude("org.apache.derby") exclude("org.apache.hbase") - // exclude("org.apache.hive", "hive-storage-api") - // exclude("org.apache.hive", "hive-service-rpc") - // exclude("org.apache.hadoop") exclude("org.apache.hive", "hive-storage-api") exclude("org.apache.hive", "hive-service-rpc") exclude("org.apache.hadoop") @@ -98,10 +87,8 @@ dependencies { exclude("org.apache.hadoop", "hadoop-yarn-server-web-proxy") exclude("org.apache.logging.log4j") exclude("org.apache.parquet", "parquet-hadoop-bundle") - // exclude("org.apache.orc") exclude("org.apache.orc") exclude("org.apache.zookeeper") - // exclude("org.datanucleus") exclude("org.datanucleus") exclude("org.eclipse.jetty.aggregate", "jetty-all") exclude("org.eclipse.jetty.orbit", "javax.servlet") @@ -113,7 +100,6 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) - testImplementation(project(":server-common")) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.sqlite.jdbc) diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index 0ee6b0ff693..577182791a2 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -34,10 +34,16 @@ dependencies { implementation(project(":api")) implementation(project(":catalogs:catalog-common")) implementation(project(":clients:client-java")) - implementation(project(":core")) - implementation(project(":common")) + implementation(project(":core")) { + exclude("*") + } + implementation(project(":common")) { + exclude("*") + } implementation(project(":iceberg:iceberg-common")) - implementation(project(":server-common")) + implementation(project(":server-common")) { + exclude("*") + } implementation(libs.bundles.iceberg) implementation(libs.bundles.jetty) implementation(libs.bundles.jersey)