[#400] test: Add Catalog-iceberg e2e integration test.

bin/common.sh

@@ -29,6 +29,30 @@ if [[ -f "${GRAVITON_CONF_DIR}/graviton-env.sh" ]]; then
 fi
 
 GRAVITON_CLASSPATH+=":${GRAVITON_CONF_DIR}"
+export DEFAULT_MYSQL_VERSION=8.0.15
+export GET_MYSQL_JAR=true
+
+function download_mysql_jdbc_jar(){
+  if [[ -z "${MYSQL_JAR_URL}" ]]; then
+    if [[ -z "${MYSQL_VERSION}" ]]; then
+      MYSQL_VERSION="${DEFAULT_MYSQL_VERSION}"
+    fi
+    MYSQL_JAR_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_VERSION}/mysql-connector-java-${MYSQL_VERSION}.jar"
+  fi
+
+  echo "Downloading mysql jdbc jar from ${MYSQL_JAR_URL}."
+  if type wget >/dev/null 2>&1; then
+    wget ${MYSQL_JAR_URL} -P "${GRAVITON_HOME}/libs" --no-check-certificate
+  elif type curl >/dev/null 2>&1; then
+    curl -o "${GRAVITON_HOME}/libs/mysql-connector-java-${MYSQL_VERSION}.jar" ${MYSQL_JAR_URL}
+  else
+    echo 'We need a tool to transfer data from or to a server. Such as wget/curl.'
+    echo 'Bye, bye!'
+    exit 1
+  fi
+
+  echo "Mysql jdbc jar is downloaded and put in the path of ${GRAVITON_HOME}/libs."
+}
 
 function check_java_version() {
   if [[ -n "${JAVA_HOME+x}" ]]; then

bin/graviton.sh

@@ -27,6 +27,29 @@ bin="$(cd "${bin}">/dev/null; pwd)"
 
 check_java_version
 
+function check_jdbc_jar() {
+  if [[ -d "${1}/libs" ]]; then
+    mysql_connector_exists=$(find -L "${1}/libs" -name "mysql-connector*")
+    if [[ -z "${mysql_connector_exists}" ]]; then
+      if [[ ${GET_MYSQL_JAR} = true ]]; then
+        download_mysql_jdbc_jar
+      else
+        echo -e "\\033[31mError: There is no mysql jdbc jar in lib.\\033[0m"
+        echo -e "\\033[31mPlease download a mysql jdbc jar and put it under lib manually.\\033[0m"
+        echo -e "\\033[31mIt would download mysql jdbc jar automatically.\\033[0m"
+        exit 1
+      fi
+    fi
+    mysql_connector_exists=$(find -L "${1}" -name "mysql-connector*")
+    if [[ -n "${mysql_connector_exists}" ]]; then
+        if [[ -d "${GRAVITON_HOME}/catalogs/lakehouse-iceberg/libs" ]]; then
+                echo "copy mysql connector to iceberg libs."
+               $(cp ${GRAVITON_HOME}/libs/mysql-connector* ${GRAVITON_HOME}/catalogs/lakehouse-iceberg/libs)
+        fi
+    fi
+  fi
+}
+
 function check_process_status() {
   local pid=$(found_graviton_server_pid)
 
@@ -83,6 +106,7 @@ function wait_for_graviton_server_to_die() {
 
 function start() {
   local pid=$(found_graviton_server_pid)
+  check_jdbc_jar "${GRAVITON_HOME}"
 
   if [[ ! -z "${pid}" ]]; then
     if kill -0 ${pid} >/dev/null 2>&1; then

catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/IcebergCatalogOperations.java

@@ -466,15 +466,27 @@ public Table createTable(
         LOG.warn("Iceberg schema (database) does not exist: {}", schemaIdent);
         throw new NoSuchSchemaException("Iceberg Schema (database) does not exist " + schemaIdent);
       }
+      IcebergColumn[] icebergColumns =
+          Arrays.stream(columns)
+              .map(
+                  column ->
+                      new IcebergColumn.Builder()
+                          .withName(column.name())
+                          .withType(column.dataType())
+                          .withComment(column.comment())
+                          .withOptional(true)
+                          .build())
+              .toArray(IcebergColumn[]::new);
 
       IcebergTable createdTable =
           new IcebergTable.Builder()
               .withName(tableIdent.name())
-              .withColumns(columns)
+              .withColumns(icebergColumns)
               .withComment(comment)
               .withPartitions(partitions)
               .withSortOrders(sortOrders)
               .withProperties(properties)
+              .withDistribution(Distribution.NONE)
               .withAuditInfo(
                   new AuditInfo.Builder()
                       .withCreator(currentUser())
@@ -522,7 +534,7 @@ private static String currentUser() {
 
   @Override
   public PropertiesMetadata tablePropertiesMetadata() throws UnsupportedOperationException {
-    return tablePropertiesMetadata;
+    return Maps::newHashMap;
   }
 
   @Override

catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/IcebergTable.java

@@ -12,6 +12,7 @@
 import com.datastrato.graviton.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper;
 import com.datastrato.graviton.catalog.rel.BaseTable;
 import com.datastrato.graviton.meta.AuditInfo;
+import com.datastrato.graviton.rel.Distribution;
 import com.google.common.collect.Maps;
 import java.util.Map;
 import lombok.Getter;
@@ -77,6 +78,7 @@ public static IcebergTable fromIcebergTable(TableMetadata table, String tableNam
         .withLocation(table.location())
         .withProperties(properties)
         .withColumns(icebergColumns)
+        .withDistribution(Distribution.NONE)
         .withName(tableName)
         .withAuditInfo(AuditInfo.EMPTY)
         .withPartitions(FromIcebergPartitionSpec.fromPartitionSpec(table.spec(), schema))

common/src/main/java/com/datastrato/graviton/dto/util/DTOConverters.java

@@ -97,7 +97,7 @@ public static TableDTO toDTO(Table table) {
   }
 
   public static DistributionDTO toDTO(Distribution distribution) {
-    if (Distribution.NONE.equals(distribution)) {
+    if (Distribution.NONE.equals(distribution) || null == distribution) {
       return DistributionDTO.NONE;
     }
 

gradle/libs.versions.toml

@@ -28,6 +28,7 @@ rocksdbjni = "7.7.3"
 iceberg = '1.3.1'
 trino = '426'
 spark = "3.4.1"
+mysql = "8.0.15"
 
 
 protobuf-plugin = "0.9.2"
@@ -91,7 +92,7 @@ trino-toolkit= { group = "io.trino", name = "trino-plugin-toolkit", version.ref
 trino-testing= { group = "io.trino", name = "trino-testing", version.ref = "trino" }
 iceberg-spark-runtime = { group = "org.apache.iceberg", name = "iceberg-spark-runtime-3.4_2.13", version.ref = "iceberg" }
 spark-sql = { group = "org.apache.spark", name = "spark-sql_2.13", version.ref = "spark" }
-
+mysql = { group = "mysql", name = "mysql-connector-java", version.ref = "mysql" }
 
 [bundles]
 log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api"]

integration-test/build.gradle.kts

@@ -89,12 +89,14 @@ dependencies {
 
   testCompileOnly(libs.lombok)
   testAnnotationProcessor(libs.lombok)
+  testImplementation(libs.iceberg.hive.metastore)
   testImplementation(libs.guava)
   testImplementation(libs.commons.lang3)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.httpclient5)
   testRuntimeOnly(libs.junit.jupiter.engine)
+  testImplementation(libs.mysql)
   testImplementation(libs.mockito.core)
   testImplementation(libs.bundles.log4j)
   testImplementation(libs.iceberg.spark.runtime)