support it

build.gradle.kts

@@ -545,8 +545,7 @@ tasks {
   }
 
   val compileIcebergRESTServer by registering {
-    dependsOn("iceberg-rest-server:copyLibs") 
-
+    dependsOn("iceberg-rest-server:copyLibs")
     group = "Iceberg REST server distribution"
     outputs.dir(projectDir.dir("distribution/${rootProject.name}-iceberg-rest-server"))
     doLast {
@@ -597,7 +596,7 @@ tasks {
   val assembleIcebergRESTServer by registering(Tar::class) {
     dependsOn("iceberg-rest-server:copyLibs")
     group = "gravitino distribution"
-    //finalizedBy("checksumTrinoConnector")
+    // finalizedBy("checksumTrinoConnector")
     into("${rootProject.name}-iceberg-rest-server-$version")
     from("iceberg-rest-server/build/libs")
     compression = Compression.GZIP
@@ -645,7 +644,7 @@ tasks {
   register("copySubprojectDependencies", Copy::class) {
     subprojects.forEach() {
       if (!it.name.startsWith("catalog") &&
-        !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg-")  && it.name != "trino-connector" &&
+        !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg-") && it.name != "trino-connector" &&
         it.name != "integration-test" && it.name != "bundled-catalog" && it.name != "flink-connector"
       ) {
         from(it.configurations.runtimeClasspath)

catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergRestIT.java

@@ -6,7 +6,6 @@
 
 import com.datastrato.gravitino.auxiliary.AuxiliaryServiceManager;
 import com.datastrato.gravitino.iceberg.common.IcebergConfig;
-import com.datastrato.gravitino.iceberg.IcebergRESTService;
 import com.datastrato.gravitino.iceberg.common.IcebergConstants;
 import com.datastrato.gravitino.integration.test.container.HiveContainer;
 import com.datastrato.gravitino.server.web.JettyServerConfig;

gradle.properties

@@ -26,4 +26,4 @@ defaultScalaVersion = 2.12
 pythonVersion = 3.8
 
 # skipDockerTests is used to skip the tests that require Docker to be running.
-skipDockerTests = true
+skipDockerTests = false

iceberg-common/build.gradle.kts

@@ -18,6 +18,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
 
 dependencies {
   implementation(project(":core"))
+  implementation(project(":common"))
   implementation(project(":server-common"))
   implementation(libs.bundles.iceberg)
   implementation(libs.bundles.log4j)
@@ -29,6 +30,29 @@ dependencies {
   }
   implementation(libs.hadoop2.hdfs)
   implementation(libs.hadoop2.mapreduce.client.core)
+  implementation(libs.hive2.metastore) {
+    exclude("co.cask.tephra")
+    exclude("com.github.spotbugs")
+    exclude("com.google.code.findbugs", "jsr305")
+    exclude("com.tdunning", "json")
+    exclude("javax.transaction", "transaction-api")
+    exclude("org.apache.avro", "avro")
+    exclude("org.apache.hbase")
+    exclude("org.apache.hadoop", "hadoop-yarn-api")
+    exclude("org.apache.hadoop", "hadoop-yarn-server-applicationhistoryservice")
+    exclude("org.apache.hadoop", "hadoop-yarn-server-common")
+    exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager")
+    exclude("org.apache.hadoop", "hadoop-yarn-server-web-proxy")
+    exclude("org.apache.logging.log4j")
+    exclude("org.apache.parquet", "parquet-hadoop-bundle")
+    exclude("org.apache.zookeeper")
+    exclude("org.eclipse.jetty.aggregate", "jetty-all")
+    exclude("org.eclipse.jetty.orbit", "javax.servlet")
+    exclude("org.pentaho") // missing dependency
+    exclude("org.slf4j", "slf4j-log4j12")
+    exclude("com.zaxxer", "HikariCP")
+    exclude("com.sun.jersey", "jersey-server")
+  }
 
   annotationProcessor(libs.lombok)
 

iceberg-common/src/main/java/com/datastrato/gravitino/iceberg/common/IcebergConfig.java

@@ -17,6 +17,7 @@
 import org.apache.commons.lang3.StringUtils;
 
 public class IcebergConfig extends Config implements OverwriteDefaultConfig {
+  public static final String ICEBERG_CONFIG_PREFIX = "gravitino.iceberg-rest.";
 
   public static final ConfigEntry<String> CATALOG_BACKEND =
       new ConfigBuilder(IcebergConstants.CATALOG_BACKEND)

iceberg-common/src/main/java/com/datastrato/gravitino/iceberg/common/IcebergConstants.java

@@ -25,5 +25,4 @@ public class IcebergConstants {
   public static final String ICEBERG_METRICS_QUEUE_CAPACITY = "metricsQueueCapacity";
 
   public static final String GRAVITINO_ICEBERG_REST_SERVICE_NAME = "iceberg-rest";
-
 }

iceberg-rest-server/build.gradle.kts

@@ -18,6 +18,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
 
 dependencies {
   implementation(project(":core"))
+  implementation(project(":common"))
   implementation(project(":server-common"))
   implementation(project(":iceberg-common"))
   implementation(libs.bundles.iceberg)
@@ -39,6 +40,8 @@ dependencies {
 
   implementation(libs.metrics.jersey2)
 
+  testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion")
+  testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
   testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
     exclude("org.apache.avro")
     exclude("org.apache.hadoop")
@@ -57,6 +60,7 @@ dependencies {
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.mockito.core)
 
+  testImplementation(libs.sqlite.jdbc)
   testImplementation(libs.slf4j.api)
   testImplementation(libs.testcontainers)
 

iceberg-rest-server/src/main/java/com/datastrato/gravitino/server/IcebergRESTServer.java

@@ -8,15 +8,19 @@
 import com.datastrato.gravitino.Config;
 import com.datastrato.gravitino.GravitinoEnv;
 import com.datastrato.gravitino.iceberg.RESTService;
+import com.datastrato.gravitino.iceberg.common.IcebergConfig;
 import com.datastrato.gravitino.server.authentication.ServerAuthenticator;
+import com.datastrato.gravitino.server.web.JettyServerConfig;
+import java.util.HashMap;
+import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class IcebergRESTServer {
 
   private static final Logger LOG = LoggerFactory.getLogger(IcebergRESTServer.class);
 
-  public static final String CONF_FILE = "iceberg-rest.conf";
+  public static final String CONF_FILE = "iceberg-rest-server.conf";
 
   private final Config serverConfig;
 
@@ -31,10 +35,21 @@ public IcebergRESTServer(Config config) {
 
   public void initialize() {
     gravitinoEnv.initialize(serverConfig, false);
-    icebergRESTService.serviceInit(serverConfig.getConfigsWithPrefix("gravitino.iceberg-rest"));
+    icebergRESTService.serviceInit(getIcebergAndServerConfigs(serverConfig));
     ServerAuthenticator.getInstance().initialize(serverConfig);
   }
 
+  private Map<String, String> getIcebergAndServerConfigs(Config config) {
+    Map<String, String> all = new HashMap<>();
+    Map<String, String> icebergProperties =
+        config.getConfigsWithPrefix(IcebergConfig.ICEBERG_CONFIG_PREFIX);
+    all.putAll(icebergProperties);
+    Map<String, String> serverProperties =
+        config.getConfigsWithPrefix(JettyServerConfig.GRAVITINO_SERVER_CONFIG_PREFIX);
+    all.putAll(serverProperties);
+    return all;
+  }
+
   public void start() {
     icebergRESTService.serviceStart();
   }

iceberg-rest-server/src/test/java/com/datastrato/gravitino/iceberg/integration/test/IcebergRESTHiveCatalogIT.java

@@ -4,10 +4,8 @@
  */
 package com.datastrato.gravitino.iceberg.integration.test;
 
-import com.datastrato.gravitino.auxiliary.AuxiliaryServiceManager;
 import com.datastrato.gravitino.iceberg.common.IcebergCatalogBackend;
 import com.datastrato.gravitino.iceberg.common.IcebergConfig;
-import com.datastrato.gravitino.iceberg.common.IcebergConstants;
 import com.datastrato.gravitino.integration.test.container.ContainerSuite;
 import com.datastrato.gravitino.integration.test.container.HiveContainer;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
@@ -17,9 +15,6 @@
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
 
-// Hive&Jdbc catalog must be tested with gravitino-docker-test env,
-// so we should create a separate class instead using junit `parameterized test`
-// to auto-generate catalog type
 @Tag("gravitino-docker-test")
 @TestInstance(Lifecycle.PER_CLASS)
 public class IcebergRESTHiveCatalogIT extends IcebergRESTServiceIT {
@@ -36,34 +31,25 @@ void initEnv() {
 
   @Override
   Map<String, String> getCatalogConfig() {
-    Map<String, String> customConfigs = new HashMap<>();
-    customConfigs.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.CATALOG_BACKEND.getKey(),
+    Map<String, String> configMap = new HashMap<>();
+    configMap.put(
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_BACKEND.getKey(),
         IcebergCatalogBackend.HIVE.toString().toLowerCase());
 
-    customConfigs.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.CATALOG_URI.getKey(),
+    configMap.put(
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_URI.getKey(),
         String.format(
             "thrift://%s:%d",
             containerSuite.getHiveContainer().getContainerIpAddress(),
             HiveContainer.HIVE_METASTORE_PORT));
 
-    customConfigs.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.CATALOG_WAREHOUSE.getKey(),
+    configMap.put(
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(),
         GravitinoITUtils.genRandomName(
             String.format(
                 "hdfs://%s:%d/user/hive/warehouse-hive",
                 containerSuite.getHiveContainer().getContainerIpAddress(),
                 HiveContainer.HDFS_DEFAULTFS_PORT)));
-    return customConfigs;
+    return configMap;
   }
 }

iceberg-rest-server/src/test/java/com/datastrato/gravitino/iceberg/integration/test/IcebergRESTJdbcCatalogIT.java

@@ -4,10 +4,8 @@
  */
 package com.datastrato.gravitino.iceberg.integration.test;
 
-import com.datastrato.gravitino.auxiliary.AuxiliaryServiceManager;
 import com.datastrato.gravitino.iceberg.common.IcebergCatalogBackend;
 import com.datastrato.gravitino.iceberg.common.IcebergConfig;
-import com.datastrato.gravitino.iceberg.common.IcebergConstants;
 import com.datastrato.gravitino.integration.test.container.ContainerSuite;
 import com.datastrato.gravitino.integration.test.container.HiveContainer;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
@@ -19,7 +17,7 @@
 
 @Tag("gravitino-docker-test")
 @TestInstance(Lifecycle.PER_CLASS)
-public class IcebergRESTJdbcCatalogIT extends IcebergConstantsIT {
+public class IcebergRESTJdbcCatalogIT extends IcebergRESTServiceIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
 
   public IcebergRESTJdbcCatalogIT() {
@@ -35,52 +33,28 @@ public Map<String, String> getCatalogConfig() {
     Map<String, String> configMap = new HashMap<>();
 
     configMap.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.CATALOG_BACKEND.getKey(),
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_BACKEND.getKey(),
         IcebergCatalogBackend.JDBC.toString().toLowerCase());
 
     configMap.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.JDBC_DRIVER.getKey(),
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.JDBC_DRIVER.getKey(),
         "org.sqlite.JDBC");
 
     configMap.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.CATALOG_URI.getKey(),
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_URI.getKey(),
         "jdbc:sqlite::memory:");
 
     configMap.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.JDBC_USER.getKey(),
-        "iceberg");
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.JDBC_USER.getKey(), "iceberg");
 
     configMap.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.JDBC_PASSWORD.getKey(),
-        "iceberg");
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.JDBC_PASSWORD.getKey(), "iceberg");
 
     configMap.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.JDBC_INIT_TABLES.getKey(),
-        "true");
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.JDBC_INIT_TABLES.getKey(), "true");
 
     configMap.put(
-        AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX
-            + IcebergConstants.GRAVITINO_ICEBERG_REST_SERVICE_NAME
-            + "."
-            + IcebergConfig.CATALOG_WAREHOUSE.getKey(),
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(),
         GravitinoITUtils.genRandomName(
             String.format(
                 "hdfs://%s:%d/user/hive/warehouse-jdbc-sqlite",

iceberg-rest-server/src/test/java/com/datastrato/gravitino/iceberg/integration/test/IcebergRESTMemoryCatalogIT.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+package com.datastrato.gravitino.iceberg.integration.test;
+
+import com.datastrato.gravitino.iceberg.common.IcebergCatalogBackend;
+import com.datastrato.gravitino.iceberg.common.IcebergConfig;
+import java.util.HashMap;
+import java.util.Map;
+
+public class IcebergRESTMemoryCatalogIT extends IcebergRESTServiceIT {
+
+  @Override
+  void initEnv() {}
+
+  @Override
+  Map<String, String> getCatalogConfig() {
+    Map<String, String> configMap = new HashMap<>();
+    configMap.put(
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_BACKEND.getKey(),
+        IcebergCatalogBackend.MEMORY.toString().toLowerCase());
+
+    configMap.put(
+        IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConfig.CATALOG_WAREHOUSE.getKey(), "/tmp/");
+    return configMap;
+  }
+}