Merge branch 'main' into #4545-Shrink-Paimon-catalog-binary-package-size

.github/workflows/backend-integration-test.yml

@@ -99,7 +99,7 @@ jobs:
         id: integrationTest
         run: >
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
-          -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
+          -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
           -x :authorizations:authorization-ranger:test
@@ -169,7 +169,7 @@ jobs:
         id: integrationTest
         run: >
           ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
-          -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test 
+          -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
           -x :authorizations:authorization-ranger:test

.github/workflows/flink-integration-test.yml

@@ -86,8 +86,8 @@ jobs:
       - name: Flink Integration Test
         id: integrationTest
         run: |
-          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
-          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v3
@@ -96,9 +96,9 @@ jobs:
           name: flink-connector-integrate-test-reports-${{ matrix.java-version }}
           path: |
             build/reports
-            flink-connector/build/flink-connector-integration-test.log
-            flink-connector/build/*.tar
+            flink-connector/flink/build/*.log
+            flink-connector/flink/build/*.tar
             distribution/package/logs/gravitino-server.out
             distribution/package/logs/gravitino-server.log
             catalogs/**/*.log
-            catalogs/**/*.tar
+            catalogs/**/*.tar

DISCLAIMER_WIP.txt

@@ -3,4 +3,4 @@ Apache Gravitino is an effort undergoing incubation at The Apache Software Found
 Some of the incubating project’s releases may not be fully compliant with ASF policy and while we have documented the licensing of all code in detail, we know that currently our release would:
 - Contains code that may not be compatible with the Apache License
 
-If you are planning to incorporate this work into your product/project, please be aware that you will need to conduct a thorough licensing review to determine the overall implications of including this work. For the current status of this project through the Apache Incubator, visit: https://incubator.apache.org/projects/Apache Podling-Name.html
+If you are planning to incorporate this work into your product/project, please be aware that you will need to conduct a thorough licensing review to determine the overall implications of including this work. For the current status of this project through the Apache Incubator, visit: https://incubator.apache.org/projects/gravitino.html

LICENSE.bin

@@ -358,6 +358,8 @@
    WildFly
    Confluent Kafka Streams Examples
    Apache Arrow
+   Rome
+   Jettison
 
    This product bundles various third-party components also under the
    Apache Software Foundation License 1.1
@@ -404,6 +406,7 @@
    Common Development and Distribution License 1.0
 
    Javax Activation
+   Javax Mail
    Stax API
    Java Servlet API
    JSR311 API

authorizations/authorization-ranger/build.gradle.kts

@@ -31,14 +31,17 @@ dependencies {
   implementation(project(":core")) {
     exclude(group = "*")
   }
+
   implementation(libs.bundles.log4j)
-  implementation(libs.commons.collections4)
   implementation(libs.commons.lang3)
   implementation(libs.guava)
-  implementation(libs.slf4j.api)
-
+  implementation(libs.javax.jaxb.api) {
+    exclude("*")
+  }
+  implementation(libs.javax.ws.rs.api)
+  implementation(libs.jettison)
   compileOnly(libs.lombok)
-  implementation(libs.jackson.annotations)
+  implementation(libs.mail)
   implementation(libs.ranger.intg) {
     exclude("org.apache.hadoop", "hadoop-common")
     exclude("org.apache.hive", "hive-storage-api")
@@ -50,13 +53,12 @@ dependencies {
     exclude("org.elasticsearch.plugin")
     exclude("org.apache.ranger", "ranger-plugins-audit")
     exclude("org.apache.ranger", "ranger-plugins-cred")
-    exclude("org.apache.ranger", "ranger-plugins-classloader")
+    exclude("org.apache.ranger", "ranger-plugin-classloader")
+    exclude("net.java.dev.jna")
     exclude("javax.ws.rs")
+    exclude("org.eclipse.jetty")
   }
-  implementation(libs.javax.ws.rs.api)
-  implementation(libs.javax.jaxb.api) {
-    exclude("*")
-  }
+  implementation(libs.rome)
 
   testImplementation(project(":common"))
   testImplementation(project(":clients:client-java"))
@@ -72,13 +74,16 @@ dependencies {
     exclude("org.apache.lucene")
     exclude("org.apache.solr")
     exclude("org.apache.kafka")
+    exclude("org.eclipse.jetty")
     exclude("org.elasticsearch")
     exclude("org.elasticsearch.client")
     exclude("org.elasticsearch.plugin")
     exclude("javax.ws.rs")
+    exclude("org.apache.ranger", "ranger-plugin-classloader")
   }
   testImplementation(libs.hive2.jdbc) {
     exclude("org.slf4j")
+    exclude("org.eclipse.jetty.aggregate")
   }
   testImplementation(libs.mysql.driver)
 }
@@ -91,7 +96,11 @@ tasks {
 
   val copyAuthorizationLibs by registering(Copy::class) {
     dependsOn("jar", "runtimeJars")
-    from("build/libs")
+    from("build/libs") {
+      exclude("guava-*.jar")
+      exclude("log4j-*.jar")
+      exclude("slf4j-*.jar")
+    }
     into("$rootDir/distribution/package/authorizations/ranger/libs")
   }
 

build.gradle.kts

@@ -571,6 +571,7 @@ tasks {
         from(projectDir.file("LICENSE.bin")) { into("package") }
         from(projectDir.file("NOTICE.bin")) { into("package") }
         from(projectDir.file("README.md")) { into("package") }
+        from(projectDir.file("DISCLAIMER_WIP.txt")) { into("package") }
         into(outputDir)
         rename { fileName ->
           fileName.replace(".bin", "")
@@ -616,6 +617,26 @@ tasks {
         from(projectDir.file("LICENSE.bin")) { into("${rootProject.name}-iceberg-rest-server") }
         from(projectDir.file("NOTICE.bin")) { into("${rootProject.name}-iceberg-rest-server") }
         from(projectDir.file("README.md")) { into("${rootProject.name}-iceberg-rest-server") }
+        from(projectDir.file("DISCLAIMER_WIP.txt")) { into("${rootProject.name}-iceberg-rest-server") }
+        into(outputDir)
+        rename { fileName ->
+          fileName.replace(".bin", "")
+        }
+      }
+    }
+  }
+
+  val compileTrinoConnector by registering {
+    dependsOn("trino-connector:copyLibs")
+    group = "gravitino distribution"
+    outputs.dir(projectDir.dir("distribution/${rootProject.name}-trino-connector"))
+    doLast {
+      copy {
+        from(projectDir.dir("licenses")) { into("${rootProject.name}-trino-connector/licenses") }
+        from(projectDir.file("LICENSE.bin")) { into("${rootProject.name}-trino-connector") }
+        from(projectDir.file("NOTICE.bin")) { into("${rootProject.name}-trino-connector") }
+        from(projectDir.file("README.md")) { into("${rootProject.name}-trino-connector") }
+        from(projectDir.file("DISCLAIMER_WIP.txt")) { into("${rootProject.name}-trino-connector") }
         into(outputDir)
         rename { fileName ->
           fileName.replace(".bin", "")
@@ -636,11 +657,11 @@ tasks {
   }
 
   val assembleTrinoConnector by registering(Tar::class) {
-    dependsOn("trino-connector:copyLibs")
+    dependsOn("compileTrinoConnector")
     group = "gravitino distribution"
     finalizedBy("checksumTrinoConnector")
     into("${rootProject.name}-trino-connector-$version")
-    from("trino-connector/build/libs")
+    from(compileTrinoConnector.map { it.outputs.files.single() })
     compression = Compression.GZIP
     archiveFileName.set("${rootProject.name}-trino-connector-$version.tar.gz")
     destinationDirectory.set(projectDir.dir("distribution"))
@@ -715,7 +736,7 @@ tasks {
       if (!it.name.startsWith("catalog") &&
         !it.name.startsWith("authorization") &&
         !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" &&
-        it.name != "integration-test" && it.name != "bundled-catalog" && it.name != "flink-connector"
+        it.name != "integration-test" && it.name != "bundled-catalog" && !it.name.startsWith("flink")
       ) {
         from(it.configurations.runtimeClasspath)
         into("distribution/package/libs")
@@ -732,9 +753,9 @@ tasks {
         !it.name.startsWith("spark") &&
         !it.name.startsWith("iceberg") &&
         !it.name.startsWith("integration-test") &&
+        !it.name.startsWith("flink") &&
         it.name != "trino-connector" &&
-        it.name != "bundled-catalog" &&
-        it.name != "flink-connector"
+        it.name != "bundled-catalog"
       ) {
         dependsOn("${it.name}:build")
         from("${it.name}/build/libs")

catalogs/catalog-hadoop/build.gradle.kts

@@ -25,21 +25,45 @@ plugins {
 }
 
 dependencies {
-  implementation(project(":api"))
-  implementation(project(":core"))
-  implementation(project(":common"))
+  implementation(project(":api")) {
+    exclude(group = "*")
+  }
+
+  implementation(project(":core")) {
+    exclude(group = "*")
+  }
+  implementation(project(":common")) {
+    exclude(group = "*")
+  }
 
-  implementation(libs.guava)
   implementation(libs.hadoop3.common) {
     exclude("com.sun.jersey")
     exclude("javax.servlet", "servlet-api")
+    exclude("org.eclipse.jetty", "*")
+    exclude("org.apache.hadoop", "hadoop-auth")
+    exclude("org.apache.curator", "curator-client")
+    exclude("org.apache.curator", "curator-framework")
+    exclude("org.apache.curator", "curator-recipes")
+    exclude("org.apache.avro", "avro")
+    exclude("com.sun.jersey", "jersey-servlet")
   }
 
   implementation(libs.hadoop3.hdfs) {
     exclude("com.sun.jersey")
     exclude("javax.servlet", "servlet-api")
+    exclude("com.google.guava", "guava")
+    exclude("commons-io", "commons-io")
+    exclude("org.eclipse.jetty", "*")
+    exclude("io.netty")
+    exclude("org.fusesource.leveldbjni")
+  }
+  implementation(libs.hadoop3.client) {
+    exclude("org.apache.hadoop", "hadoop-mapreduce-client-core")
+    exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient")
+    exclude("org.apache.hadoop", "hadoop-yarn-api")
+    exclude("org.apache.hadoop", "hadoop-yarn-client")
+    exclude("com.squareup.okhttp", "okhttp")
   }
-  implementation(libs.hadoop3.client)
 
   implementation(libs.slf4j.api)
 
@@ -71,7 +95,18 @@ tasks {
 
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn("jar", "runtimeJars")
-    from("build/libs")
+    from("build/libs") {
+      exclude("slf4j-*.jar")
+      exclude("guava-*.jar")
+      exclude("curator-*.jar")
+      exclude("netty-*.jar")
+      exclude("snappy-*.jar")
+      exclude("zookeeper-*.jar")
+      exclude("jetty-*.jar")
+      exclude("javax.servlet-*.jar")
+      exclude("kerb-*.jar")
+      exclude("kerby-*.jar")
+    }
     into("$rootDir/distribution/package/catalogs/hadoop/libs")
   }
 

catalogs/catalog-hive/build.gradle.kts

@@ -30,32 +30,53 @@ val icebergVersion: String = libs.versions.iceberg.get()
 val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()
 
 dependencies {
-  implementation(project(":api"))
-  implementation(project(":catalogs:catalog-common"))
-  implementation(project(":core"))
+  implementation(project(":api")) {
+    exclude("*")
+  }
+  implementation(project(":catalogs:catalog-common")) {
+    exclude("*")
+  }
+  implementation(project(":core")) {
+    exclude("*")
+  }
 
   implementation(libs.caffeine)
+  implementation(libs.commons.collections3)
+  implementation(libs.commons.configuration1)
+  implementation(libs.htrace.core4)
   implementation(libs.guava)
+  implementation(libs.hadoop2.auth) {
+    exclude("*")
+  }
   implementation(libs.hive2.exec) {
     artifact {
       classifier = "core"
     }
     exclude("com.google.code.findbugs", "jsr305")
     exclude("com.google.protobuf")
     exclude("org.apache.avro")
+    exclude("org.apache.ant")
     exclude("org.apache.calcite")
     exclude("org.apache.calcite.avatica")
     exclude("org.apache.curator")
+    exclude("org.apache.derby")
     exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager")
+    exclude("org.apache.hive", "hive-llap-tez")
+    exclude("org.apache.hive", "hive-vector-code-gen")
+    exclude("org.apache.ivy")
     exclude("org.apache.logging.log4j")
     exclude("org.apache.zookeeper")
+    exclude("org.codehaus.groovy", "groovy-all")
+    exclude("org.datanucleus", "datanucleus-core")
     exclude("org.eclipse.jetty.aggregate", "jetty-all")
     exclude("org.eclipse.jetty.orbit", "javax.servlet")
     exclude("org.openjdk.jol")
     exclude("org.pentaho")
     exclude("org.slf4j")
   }
+  implementation(libs.woodstox.core)
   implementation(libs.hive2.metastore) {
+    exclude("ant")
     exclude("co.cask.tephra")
     exclude("com.github.joshelser")
     exclude("com.google.code.findbugs", "jsr305")
@@ -64,13 +85,16 @@ dependencies {
     exclude("com.zaxxer", "HikariCP")
     exclude("io.dropwizard.metricss")
     exclude("javax.transaction", "transaction-api")
+    exclude("org.apache.ant")
     exclude("org.apache.avro")
     exclude("org.apache.curator")
+    exclude("org.apache.derby")
     exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager")
     exclude("org.apache.hbase")
     exclude("org.apache.logging.log4j")
     exclude("org.apache.parquet", "parquet-hadoop-bundle")
     exclude("org.apache.zookeeper")
+    exclude("org.datanucleus")
     exclude("org.eclipse.jetty.aggregate", "jetty-all")
     exclude("org.eclipse.jetty.orbit", "javax.servlet")
     exclude("org.openjdk.jol")
@@ -135,7 +159,11 @@ tasks {
 
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn("jar", "runtimeJars")
-    from("build/libs")
+    from("build/libs") {
+      exclude("guava-*.jar")
+      exclude("log4j-*.jar")
+      exclude("slf4j-*.jar")
+    }
     into("$rootDir/distribution/package/catalogs/hive/libs")
   }
 

catalogs/catalog-hive/src/main/java/org/apache/gravitino/catalog/hive/HiveTable.java

@@ -31,7 +31,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import lombok.ToString;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.gravitino.catalog.hive.converter.HiveDataTypeConverter;
 import org.apache.gravitino.connector.BaseTable;
@@ -87,7 +86,7 @@ public static HiveTable.Builder fromHiveTable(Table table) {
 
     StorageDescriptor sd = table.getSd();
     Distribution distribution = Distributions.NONE;
-    if (CollectionUtils.isNotEmpty(sd.getBucketCols())) {
+    if (sd.getBucketCols() != null && !sd.getBucketCols().isEmpty()) {
       // Hive table use hash strategy as bucketing strategy
       distribution =
           Distributions.hash(
@@ -96,7 +95,7 @@ public static HiveTable.Builder fromHiveTable(Table table) {
     }
 
     SortOrder[] sortOrders = new SortOrder[0];
-    if (CollectionUtils.isNotEmpty(sd.getSortCols())) {
+    if (sd.getSortCols() != null && !sd.getSortCols().isEmpty()) {
       sortOrders =
           sd.getSortCols().stream()
               .map(