Merge branch 'main' into apache#4545-Shrink-Paimon-catalog-binary-pac…

…kage-size

.github/workflows/backend-integration-test.yml

@@ -108,8 +108,7 @@ jobs:
             iceberg/iceberg-rest-server/build/*.log
             integration-test/build/*.log
             integration-test/build/*.tar
-            integration-test/build/trino-ci-container-log/hive/*.*
-            integration-test/build/trino-ci-container-log/hdfs/*.*
+            integration-test/build/trino-ci-container-log
             distribution/package/logs/*.out
             distribution/package/logs/*.log
             catalogs/**/*.log

.github/workflows/build.yml

@@ -68,6 +68,39 @@ jobs:
       - name: Build with Gradle
         run: ./gradlew build -x test -PjdkVersion=8
 
+  # To check the spark-connector is compatible with scala2.13
+  spark-connector-build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: changes
+    if: needs.changes.outputs.source_changes == 'true'
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 8
+          distribution: 'temurin'
+          cache: 'gradle'
+
+      - name: Free up disk space
+        run: |
+          dev/ci/util_free_space.sh
+
+      - name: Build with Scala2.13
+        run: |
+          ./gradlew :spark-connector:spark-3.4:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false
+          ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false
+
+      - name: Upload unit tests report
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: unit test report
+          path: |
+            build/reports
+            spark-connector/**/*.log
+
   build:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest

.github/workflows/cron-integration-test.yml

@@ -96,8 +96,7 @@ jobs:
             iceberg/iceberg-rest-server/build/*.log
             integration-test/build/*.log
             integration-test/build/*.tar
-            integration-test/build/trino-ci-container-log/hive/*.*
-            integration-test/build/trino-ci-container-log/hdfs/*.*
+            integration-test/build/trino-ci-container-log
             distribution/package/logs/*.out
             distribution/package/logs/*.log
             catalogs/**/*.log

.github/workflows/docker-image.yml

@@ -15,6 +15,7 @@ on:
           - 'gravitino-ci-trino'
           - 'gravitino-ci-doris'
           - 'gravitino-ci-ranger'
+          - 'gravitino-iceberg-rest-server'
           - 'trino'
           - 'hive'
           - 'ranger'
@@ -64,6 +65,9 @@ jobs:
           elif [ "${{ github.event.inputs.image }}" == "ranger" ]; then
             echo "image_type=ranger" >> $GITHUB_ENV
             echo "image_name=datastrato/ranger" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-iceberg-rest-server" ]; then
+            echo "image_type=iceberg-rest-server" >> $GITHUB_ENV
+            echo "image_name=datastrato/gravitino-iceberg-rest-server" >> $GITHUB_ENV
           fi
 
       - name: Check publish Docker token

api/src/main/java/org/apache/gravitino/rel/TableChange.java

@@ -20,6 +20,7 @@
 
 package org.apache.gravitino.rel;
 
+import com.google.common.base.Preconditions;
 import java.util.Arrays;
 import java.util.Objects;
 import org.apache.gravitino.annotation.Evolving;
@@ -1353,6 +1354,8 @@ final class UpdateColumnPosition implements ColumnChange {
     private final ColumnPosition position;
 
     private UpdateColumnPosition(String[] fieldName, ColumnPosition position) {
+      Preconditions.checkArgument(
+          position != ColumnPosition.defaultPos(), "Position cannot be DEFAULT");
       this.fieldName = fieldName;
       this.position = position;
     }

api/src/test/java/org/apache/gravitino/TestTableChange.java

@@ -43,6 +43,7 @@
 import org.apache.gravitino.rel.expressions.literals.Literals;
 import org.apache.gravitino.rel.types.Type;
 import org.apache.gravitino.rel.types.Types;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 public class TestTableChange {
@@ -168,6 +169,17 @@ public void testUpdateNestedColumnPosition() {
     assertEquals(newPosition, updateColumnPosition.getPosition());
   }
 
+  @Test
+  public void testDoesNotAllowDefaultInColumnPosition() {
+    String[] fieldName = {"Name", "Last Name"};
+    ColumnPosition newPosition = TableChange.ColumnPosition.defaultPos();
+    Exception exception =
+        Assertions.assertThrowsExactly(
+            IllegalArgumentException.class,
+            () -> TableChange.updateColumnPosition(fieldName, newPosition));
+    assertEquals("Position cannot be DEFAULT", exception.getMessage());
+  }
+
   @Test
   public void testUpdateColumnComment() {
     String[] fieldName = {"First Name"};

bin/common.sh

@@ -105,3 +105,20 @@ if [[ -n "${JAVA_HOME}" ]]; then
 else
   export JAVA_RUNNER=java
 fi
+
+function printArt() {
+  local lineLength=70
+  local versionText="Version: ${GRAVITINO_VERSION}  "
+  local versionTextLength=${#versionText}
+  local paddingLength=$((lineLength - versionTextLength - 3))
+  local versionLine=$(printf "#%${paddingLength}s%s#" "" "$versionText")
+
+  echo "#####################################################################"
+  echo "#    ____   ____      _ ___     ___ ___  _____  ___  _   _   ___    #"
+  echo "#   / ___| |  _ \    / \ \ \   / / |_ _||_   _||_ _|| \ | | / _ \   #"
+  echo "#  | | __  | |_) |  / _ \ \ \ / /   | |   | |   | | |  \| || | | |  #"
+  echo "#  | |_| | |  _ <  / ___ \ \ V /    | |   | |   | | | |\  || |_| |  #"
+  echo "#   \____| |_| \_\/_/   \_\ \_/    |___|  |_|  |___||_| \_| \___/   #"
+  echo "$versionLine"
+  echo "#####################################################################"
+}

bin/gravitino-iceberg-rest-server.sh

@@ -47,6 +47,7 @@ function check_process_status() {
   if [[ -z "${pid}" ]]; then
     echo "GravitinoIcebergRESTServer is not running"
   else
+    printArt
     echo "GravitinoIcebergRESTServer is running[PID:$pid]"
   fi
 }
@@ -116,8 +117,6 @@ function start() {
   if [[ -z "${pid}" ]]; then
     echo "GravitinoIcebergRESTServer start error!"
     return 1;
-  else
-    echo "GravitinoIcebergRESTServer start success!"
   fi
 
   sleep 2

bin/gravitino.sh

@@ -47,6 +47,7 @@ function check_process_status() {
   if [[ -z "${pid}" ]]; then
     echo "Gravitino Server is not running"
   else
+    printArt
     echo "Gravitino Server is running[PID:$pid]"
   fi
 }
@@ -116,8 +117,6 @@ function start() {
   if [[ -z "${pid}" ]]; then
     echo "Gravitino Server start error!"
     return 1;
-  else
-    echo "Gravitino Server start success!"
   fi
 
   sleep 2

build.gradle.kts

@@ -492,7 +492,7 @@ tasks.rat {
     "ROADMAP.md",
     "clients/client-python/.pytest_cache/*",
     "clients/client-python/.venv/*",
-    "clients/client-python/gravitino.egg-info/*",
+    "clients/client-python/apache_gravitino.egg-info/*",
     "clients/client-python/gravitino/utils/exceptions.py",
     "clients/client-python/gravitino/utils/http_client.py",
     "clients/client-python/tests/unittests/htmlcov/*",
@@ -545,6 +545,13 @@ tasks {
         rename { fileName ->
           fileName.replace(".template", "")
         }
+        eachFile {
+          if (name == "gravitino-env.sh") {
+            filter { line ->
+              line.replace("GRAVITINO_VERSION_PLACEHOLDER", "$version")
+            }
+          }
+        }
         fileMode = 0b111101101
       }
       copy {
@@ -571,7 +578,7 @@ tasks {
     doLast {
       copy {
         from(projectDir.dir("conf")) {
-          include("${rootProject.name}-iceberg-rest-server.conf.template", "log4j2.properties.template")
+          include("${rootProject.name}-iceberg-rest-server.conf.template", "${rootProject.name}-env.sh.template", "log4j2.properties.template")
           into("${rootProject.name}-iceberg-rest-server/conf")
         }
         from(projectDir.dir("bin")) {
@@ -582,6 +589,13 @@ tasks {
         rename { fileName ->
           fileName.replace(".template", "")
         }
+        eachFile {
+          if (name == "gravitino-env.sh") {
+            filter { line ->
+              line.replace("GRAVITINO_VERSION_PLACEHOLDER", "$version")
+            }
+          }
+        }
         fileMode = 0b111101101
       }
 

catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java

@@ -63,4 +63,8 @@ public class IcebergConstants {
   public static final String ICEBERG_METRICS_QUEUE_CAPACITY = "metricsQueueCapacity";
 
   public static final String GRAVITINO_ICEBERG_REST_SERVICE_NAME = "iceberg-rest";
+
+  public static final String ICEBERG_REST_CATALOG_PROVIDER = "catalog-provider";
+
+  public static final String GRAVITINO_DEFAULT_CATALOG = "__gravitino_default_catalog";
 }

catalogs/catalog-hadoop/build.gradle.kts

@@ -94,6 +94,8 @@ tasks {
     exclude { details ->
       details.file.isDirectory()
     }
+
+    fileMode = 0b111101101
   }
 
   register("copyLibAndConfig", Copy::class) {

catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/authentication/kerberos/KerberosClient.java

@@ -65,9 +65,8 @@ public String login(String keytabFilePath) throws IOException {
 
     // Login
     UserGroupInformation.setConfiguration(hadoopConf);
-    UserGroupInformation kerberosLoginUgi =
-        UserGroupInformation.loginUserFromKeytabAndReturnUGI(catalogPrincipal, keytabFilePath);
-    UserGroupInformation.setLoginUser(kerberosLoginUgi);
+    UserGroupInformation.loginUserFromKeytab(catalogPrincipal, keytabFilePath);
+    UserGroupInformation kerberosLoginUgi = UserGroupInformation.getLoginUser();
 
     // Refresh the cache if it's out of date.
     if (refreshCredentials) {

catalogs/catalog-hive/build.gradle.kts

@@ -157,6 +157,8 @@ tasks {
     exclude { details ->
       details.file.isDirectory()
     }
+
+    fileMode = 0b111101101
   }
 
   register("copyLibAndConfig", Copy::class) {

catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/JdbcCatalogOperations.java

@@ -22,6 +22,8 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
 import java.sql.Driver;
 import java.sql.DriverManager;
 import java.sql.SQLException;
@@ -54,6 +56,7 @@
 import org.apache.gravitino.connector.CatalogOperations;
 import org.apache.gravitino.connector.HasPropertyMetadata;
 import org.apache.gravitino.connector.SupportsSchemas;
+import org.apache.gravitino.exceptions.GravitinoRuntimeException;
 import org.apache.gravitino.exceptions.NoSuchCatalogException;
 import org.apache.gravitino.exceptions.NoSuchSchemaException;
 import org.apache.gravitino.exceptions.NoSuchTableException;
@@ -98,6 +101,20 @@ public class JdbcCatalogOperations implements CatalogOperations, SupportsSchemas
 
   private final JdbcColumnDefaultValueConverter columnDefaultValueConverter;
 
+  public static class JDBCDriverInfo {
+    public String name;
+    public String version;
+    public int majorVersion;
+    public int minorVersion;
+
+    public JDBCDriverInfo(String driverName, String version, int majorVersion, int minorVersion) {
+      this.name = driverName;
+      this.version = version;
+      this.majorVersion = majorVersion;
+      this.minorVersion = minorVersion;
+    }
+  }
+
   /**
    * Constructs a new instance of JdbcCatalogOperations.
    *
@@ -148,6 +165,8 @@ public void initialize(
 
     JdbcConfig jdbcConfig = new JdbcConfig(resultConf);
     this.dataSource = DataSourceUtils.createDataSource(jdbcConfig);
+
+    checkJDBCDriverVersion();
     this.databaseOperation.initialize(dataSource, exceptionConverter, resultConf);
     this.tableOperation.initialize(
         dataSource, exceptionConverter, jdbcTypeConverter, columnDefaultValueConverter, resultConf);
@@ -486,6 +505,35 @@ private Table renameTable(NameIdentifier tableIdent, TableChange.RenameTable ren
     return loadTable(NameIdentifier.of(tableIdent.namespace(), renameTable.getNewName()));
   }
 
+  /**
+   * Get the JDBC driver name and version
+   *
+   * @return Returns the JDBC driver info
+   */
+  public JDBCDriverInfo getDiverInfo() {
+    try (Connection conn = dataSource.getConnection()) {
+      DatabaseMetaData metaData = conn.getMetaData();
+      return new JDBCDriverInfo(
+          metaData.getDriverName(),
+          metaData.getDriverVersion(),
+          metaData.getDriverMajorVersion(),
+          metaData.getDriverMinorVersion());
+    } catch (final SQLException se) {
+      throw new GravitinoRuntimeException(
+          se, "Failed to get JDBC driver information %s: ", se.getMessage());
+    }
+  }
+
+  /**
+   * Check the version of JDBC driver can supported.
+   *
+   * @return Returns the result of checking the jdbc driver version. If success return true,
+   *     otherwise throw a RuntimeException
+   */
+  public boolean checkJDBCDriverVersion() {
+    return true;
+  }
+
   private Table internalAlterTable(NameIdentifier tableIdent, TableChange... changes)
       throws NoSuchTableException, IllegalArgumentException {
     String databaseName = NameIdentifier.of(tableIdent.namespace().levels()).name();

catalogs/catalog-jdbc-common/src/main/java/org/apache/gravitino/catalog/jdbc/MySQLProtocolCompatibleCatalogOperations.java

@@ -33,6 +33,8 @@ public class MySQLProtocolCompatibleCatalogOperations extends JdbcCatalogOperati
   private static final Logger LOG =
       LoggerFactory.getLogger(MySQLProtocolCompatibleCatalogOperations.class);
 
+  private static final int MYSQL_JDBC_DRIVER_MINIMAL_SUPPORT_VERSION = 8;
+
   public MySQLProtocolCompatibleCatalogOperations(
       JdbcExceptionConverter exceptionConverter,
       JdbcTypeConverter jdbcTypeConverter,
@@ -47,6 +49,18 @@ public MySQLProtocolCompatibleCatalogOperations(
         columnDefaultValueConverter);
   }
 
+  @Override
+  public boolean checkJDBCDriverVersion() {
+    JDBCDriverInfo driverInfo = getDiverInfo();
+    if (driverInfo.majorVersion < MYSQL_JDBC_DRIVER_MINIMAL_SUPPORT_VERSION) {
+      throw new RuntimeException(
+          String.format(
+              "Mysql catalog does not support the jdbc driver version %s, minimal required version is 8.0",
+              driverInfo.version));
+    }
+    return true;
+  }
+
   @Override
   public void close() {
     super.close();

catalogs/catalog-jdbc-doris/build.gradle.kts

@@ -74,6 +74,8 @@ tasks {
     exclude { details ->
       details.file.isDirectory()
     }
+
+    fileMode = 0b111101101
   }
 
   register("copyLibAndConfig", Copy::class) {

catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java

@@ -57,7 +57,7 @@
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
-@Tag("gravitino-docker-it")
+@Tag("gravitino-docker-test")
 public class TestDorisTablePartitionOperations extends TestDoris {
   private static final String databaseName = GravitinoITUtils.genRandomName("doris_test_db");
   private static final Integer DEFAULT_BUCKET_SIZE = 1;

catalogs/catalog-jdbc-mysql/build.gradle.kts

@@ -76,6 +76,8 @@ tasks {
     exclude { details ->
       details.file.isDirectory()
     }
+
+    fileMode = 0b111101101
   }
 
   register("copyLibAndConfig", Copy::class) {