xx

apache · Sep 6, 2024 · 8904232 · 8904232
1 parent f258297
commit 8904232
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 46 deletions.
diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -32,26 +32,25 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
 
 dependencies {
   implementation(project(":api")) {
-	exclude("*")
+    exclude("*")
   }
   implementation(project(":catalogs:catalog-common"))
   implementation(project(":common")) {
-	exclude("*")
+    exclude("*")
   }
   implementation(project(":core")) {
-	exclude("*")
+    exclude("*")
   }
   implementation(project(":iceberg:iceberg-common"))
   implementation(libs.bundles.iceberg)
-  implementation(libs.bundles.jersey)
-  implementation(libs.bundles.jetty)
+
   implementation(libs.bundles.log4j)
   implementation(libs.cglib)
   implementation(libs.commons.collections4)
   implementation(libs.commons.io)
   implementation(libs.commons.lang3)
   implementation(libs.guava)
-  implementation(libs.sqlite.jdbc)
+  // implementation(libs.sqlite.jdbc)
 
   annotationProcessor(libs.lombok)
 
@@ -65,7 +64,10 @@ dependencies {
 
   testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion")
   testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
-  testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") {
+    //  exclude("org.apache.hadoop")
+    exclude("org.apache.hive")
+  }
   testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
     exclude("org.apache.avro")
     exclude("org.apache.hadoop")
@@ -74,22 +76,15 @@ dependencies {
     exclude("org.rocksdb")
   }
 
-  testImplementation(libs.hadoop2.common) {
-    exclude("com.github.spotbugs")
-  }
-  testImplementation(libs.jersey.test.framework.core) {
-    exclude(group = "org.junit.jupiter")
-  }
-  testImplementation(libs.jersey.test.framework.provider.jetty) {
-    exclude(group = "org.junit.jupiter")
-  }
+  testImplementation(libs.bundles.jersey)
+  testImplementation(libs.bundles.jetty)
+  // testImplementation(libs.commons.lang3)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.mockito.core)
   // For test TestMultipleJDBCLoad, it was depended on testcontainers.mysql and testcontainers.postgresql
   testImplementation(libs.mysql.driver)
   testImplementation(libs.postgresql.driver)
-
   testImplementation(libs.slf4j.api)
   testImplementation(libs.testcontainers)
   testImplementation(libs.testcontainers.mysql)
@@ -110,7 +105,11 @@ tasks {
 
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn("jar", "runtimeJars")
-    from("build/libs")
+    from("build/libs") {
+      exclude("guava-*.jar")
+      exclude("log4j-*.jar")
+      exclude("slf4j-*.jar")
+    }
     into("$rootDir/distribution/package/catalogs/lakehouse-iceberg/libs")
   }
 
@@ -158,6 +157,11 @@ tasks.test {
   }
 }
 
+tasks.register<Copy>("copy") {
+  from(configurations.testRuntimeClasspath)
+  into("build/libs-runtime")
+}
+
 tasks.clean {
   delete("spark-warehouse")
 }

diff --git a/...c/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java b/...c/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java
@@ -26,7 +26,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.commons.lang.math.RandomUtils;
+import java.util.Random;
 import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergColumn;
 import org.apache.gravitino.rel.Column;
 import org.apache.gravitino.rel.expressions.Expression;
@@ -54,6 +54,8 @@ public class TestBaseConvert {
   protected static final Map<String, Type> GRAVITINO_TYPE = new HashMap<>();
   protected static final Map<String, org.apache.iceberg.types.Type> ICEBERG_TYPE = new HashMap<>();
 
+  private static Random random = new Random(System.currentTimeMillis());
+
   static {
     GRAVITINO_TYPE.put("BOOLEAN", org.apache.gravitino.rel.types.Types.BooleanType.get());
     // Types not supported by iceberg
@@ -113,24 +115,24 @@ protected static SortOrder[] createSortOrder(String... colNames) {
       results.add(
           SortOrders.of(
               field(colName),
-              RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING,
-              RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST));
+              random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING,
+              random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST));
     }
     return results.toArray(new SortOrder[0]);
   }
 
   protected static SortOrder createSortOrder(String name, String colName) {
     return SortOrders.of(
         FunctionExpression.of(name, field(colName)),
-        RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING,
-        RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST);
+        random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING,
+        random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST);
   }
 
   protected static SortOrder createSortOrder(String name, int width, String colName) {
     return SortOrders.of(
         FunctionExpression.of(name, Literals.integerLiteral(width), field(colName)),
-        RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING,
-        RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST);
+        random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING,
+        random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST);
   }
 
   protected static Types.NestedField createNestedField(
@@ -143,7 +145,7 @@ protected static Types.NestedField[] createNestedField(String... colNames) {
     for (int i = 0; i < colNames.length; i++) {
       results.add(
           Types.NestedField.of(
-              i + 1, RandomUtils.nextBoolean(), colNames[i], getRandomIcebergType(), TEST_COMMENT));
+              i + 1, random.nextBoolean(), colNames[i], getRandomIcebergType(), TEST_COMMENT));
     }
     return results.toArray(new Types.NestedField[0]);
   }
@@ -211,15 +213,15 @@ protected static String getGravitinoSortOrderExpressionString(Expression sortOrd
   private static Type getRandomGravitinoType() {
     Collection<Type> values = GRAVITINO_TYPE.values();
     return values.stream()
-        .skip(RandomUtils.nextInt(values.size()))
+        .skip(random.nextInt(values.size()))
         .findFirst()
         .orElseThrow(() -> new RuntimeException("No type found"));
   }
 
   private static org.apache.iceberg.types.Type getRandomIcebergType() {
     Collection<org.apache.iceberg.types.Type> values = ICEBERG_TYPE.values();
     return values.stream()
-        .skip(RandomUtils.nextInt(values.size()))
+        .skip(random.nextInt(values.size()))
         .findFirst()
         .orElseThrow(() -> new RuntimeException("No type found"));
   }

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -137,6 +137,7 @@ hive2-common = { group = "org.apache.hive", name = "hive-common", version.ref =
 hive2-jdbc = { group = "org.apache.hive", name = "hive-jdbc", version.ref = "hive2"}
 hadoop2-auth = { group = "org.apache.hadoop", name = "hadoop-auth", version.ref = "hadoop2" }
 hadoop2-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop2" }
+hadoop2-hdfs-client = { group = "org.apache.hadoop", name = "hadoop-hdfs-client", version.ref = "hadoop2" }
 hadoop2-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop2"}
 hadoop2-mapreduce-client-core = { group = "org.apache.hadoop", name = "hadoop-mapreduce-client-core", version.ref = "hadoop2"}
 hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop3" }

diff --git a/iceberg/iceberg-common/build.gradle.kts b/iceberg/iceberg-common/build.gradle.kts
@@ -51,7 +51,7 @@ dependencies {
     exclude("org.apache.zookeeper")
     exclude("org.mortbay.jetty")
   }
-  implementation(libs.hadoop2.hdfs) {
+  implementation(libs.hadoop2.hdfs.client) {
     exclude("com.sun.jersey")
     exclude("javax.servlet")
     exclude("org.fusesource.leveldbjni")
@@ -60,9 +60,6 @@ dependencies {
   implementation(libs.hadoop2.mapreduce.client.core) {
 //    exclude("*")
     exclude("*")
-    exclude("com.sun.jersey")
-    exclude("javax.servlet")
-    exclude("org.mortbay.jetty")
   }
   implementation(libs.hive2.metastore) {
     exclude("co.cask.tephra")
@@ -71,23 +68,15 @@ dependencies {
     exclude("com.sun.jersey")
     exclude("com.tdunning", "json")
     exclude("com.zaxxer", "HikariCP")
-    // exclude("com.github.joshelser")
-    // exclude("io.dropwizard.metrics")
     exclude("com.github.joshelser")
     exclude("io.dropwizard.metrics")
     exclude("javax.servlet")
     exclude("javax.transaction", "transaction-api")
-    // exclude("org.apache.ant")
     exclude("org.apache.ant")
     exclude("org.apache.avro", "avro")
-    // exclude("org.apache.curator")
-    // exclude("org.apache.derby")
     exclude("org.apache.curator")
     exclude("org.apache.derby")
     exclude("org.apache.hbase")
-    // exclude("org.apache.hive", "hive-storage-api")
-    // exclude("org.apache.hive", "hive-service-rpc")
-    // exclude("org.apache.hadoop")
     exclude("org.apache.hive", "hive-storage-api")
     exclude("org.apache.hive", "hive-service-rpc")
     exclude("org.apache.hadoop")
@@ -98,10 +87,8 @@ dependencies {
     exclude("org.apache.hadoop", "hadoop-yarn-server-web-proxy")
     exclude("org.apache.logging.log4j")
     exclude("org.apache.parquet", "parquet-hadoop-bundle")
-    // exclude("org.apache.orc")
     exclude("org.apache.orc")
     exclude("org.apache.zookeeper")
-    // exclude("org.datanucleus")
     exclude("org.datanucleus")
     exclude("org.eclipse.jetty.aggregate", "jetty-all")
     exclude("org.eclipse.jetty.orbit", "javax.servlet")
@@ -113,7 +100,6 @@ dependencies {
   annotationProcessor(libs.lombok)
   compileOnly(libs.lombok)
 
-  testImplementation(project(":server-common"))
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.sqlite.jdbc)

diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts
@@ -34,10 +34,16 @@ dependencies {
   implementation(project(":api"))
   implementation(project(":catalogs:catalog-common"))
   implementation(project(":clients:client-java"))
-  implementation(project(":core"))
-  implementation(project(":common"))
+  implementation(project(":core")) {
+    exclude("*")
+  }
+  implementation(project(":common")) {
+    exclude("*")
+  }
   implementation(project(":iceberg:iceberg-common"))
-  implementation(project(":server-common"))
+  implementation(project(":server-common")) {
+    exclude("*")
+  }
   implementation(libs.bundles.iceberg)
   implementation(libs.bundles.jetty)
   implementation(libs.bundles.jersey)