[#2552] feat(catalog-jdbc-doris): add Doris container implement for integration-test

.github/workflows/backend-integration-test.yml

@@ -55,7 +55,7 @@ jobs:
     needs: changes
     if: needs.changes.outputs.source_changes == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 60
     strategy:
       matrix:
         architecture: [linux/amd64]

catalogs/catalog-jdbc-doris/build.gradle.kts

@@ -22,6 +22,19 @@ dependencies {
   implementation(libs.guava)
   implementation(libs.jsqlparser)
   implementation(libs.slf4j.api)
+
+  testImplementation(project(":catalogs:catalog-jdbc-common", "testArtifacts"))
+  testImplementation(project(":integration-test-common", "testArtifacts"))
+
+  testImplementation(libs.commons.lang3)
+  testImplementation(libs.guava)
+  testImplementation(libs.junit.jupiter.api)
+  testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.mysql.driver)
+  testImplementation(libs.testcontainers)
+  testImplementation(libs.testcontainers.mysql)
+
+  testRuntimeOnly(libs.junit.jupiter.engine)
 }
 
 tasks {
@@ -65,6 +78,10 @@ tasks.test {
   } else {
     dependsOn(tasks.jar)
 
+    doFirst {
+      environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.1")
+    }
+
     val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
     init(this)
   }

catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisAbstractIT.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.catalog.doris.integration.test;
+
+import com.datastrato.gravitino.catalog.doris.converter.DorisColumnDefaultValueConverter;
+import com.datastrato.gravitino.catalog.doris.converter.DorisExceptionConverter;
+import com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter;
+import com.datastrato.gravitino.catalog.doris.operation.DorisDatabaseOperations;
+import com.datastrato.gravitino.catalog.doris.operation.DorisTableOperations;
+import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig;
+import com.datastrato.gravitino.catalog.jdbc.integration.test.TestJdbcAbstractIT;
+import com.datastrato.gravitino.catalog.jdbc.utils.DataSourceUtils;
+import com.datastrato.gravitino.integration.test.container.ContainerSuite;
+import com.datastrato.gravitino.integration.test.container.DorisContainer;
+import com.google.common.collect.Maps;
+import java.util.Collections;
+import java.util.Map;
+import javax.sql.DataSource;
+import org.junit.jupiter.api.BeforeAll;
+
+public class TestDorisAbstractIT extends TestJdbcAbstractIT {
+  private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
+  protected static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver";
+
+  @BeforeAll
+  public static void startup() {
+    containerSuite.startDorisContainer();
+
+    DataSource dataSource = DataSourceUtils.createDataSource(getDorisCatalogProperties());
+
+    DATABASE_OPERATIONS = new DorisDatabaseOperations();
+    TABLE_OPERATIONS = new DorisTableOperations();
+    JDBC_EXCEPTION_CONVERTER = new DorisExceptionConverter();
+    DATABASE_OPERATIONS.initialize(dataSource, JDBC_EXCEPTION_CONVERTER, Collections.emptyMap());
+    TABLE_OPERATIONS.initialize(
+        dataSource,
+        JDBC_EXCEPTION_CONVERTER,
+        new DorisTypeConverter(),
+        new DorisColumnDefaultValueConverter(),
+        Collections.emptyMap());
+  }
+
+  private static Map<String, String> getDorisCatalogProperties() {
+    Map<String, String> catalogProperties = Maps.newHashMap();
+
+    DorisContainer dorisContainer = containerSuite.getDorisContainer();
+
+    String jdbcUrl =
+        String.format(
+            "jdbc:mysql://%s:%d/",
+            dorisContainer.getContainerIpAddress(), DorisContainer.FE_MYSQL_PORT);
+
+    catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl);
+    catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), DRIVER_CLASS_NAME);
+    catalogProperties.put(JdbcConfig.USERNAME.getKey(), DorisContainer.USER_NAME);
+    catalogProperties.put(JdbcConfig.PASSWORD.getKey(), DorisContainer.PASSWORD);
+
+    return catalogProperties;
+  }
+}

catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisDatabaseOperations.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.catalog.doris.integration.test;
+
+import com.datastrato.gravitino.utils.RandomNameUtils;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag("gravitino-docker-it")
+public class TestDorisDatabaseOperations extends TestDorisAbstractIT {
+
+  @Test
+  public void testBaseOperationDatabase() {
+    String databaseName = RandomNameUtils.genRandomName("ct_db");
+    Map<String, String> properties = new HashMap<>();
+    String databaseComment = "test_comment";
+
+    Assertions.assertThrows(
+        UnsupportedOperationException.class,
+        () -> {
+          DATABASE_OPERATIONS.create(databaseName, databaseComment, properties);
+        });
+  }
+}

catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties

@@ -0,0 +1,33 @@
+#
+# Copyright 2024 Datastrato Pvt Ltd.
+# This software is licensed under the Apache License version 2.
+#
+
+# Set to debug or trace if log4j initialization is failing
+status = info
+
+# Name of the configuration
+name = ConsoleLogConfig
+
+# Console appender configuration
+appender.console.type = Console
+appender.console.name = consoleLogger
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+# Log files location
+property.logPath = ${sys:gravitino.log.path:-catalog-jdbc-doris/build/doris-integration-test.log}
+
+# File appender configuration
+appender.file.type = File
+appender.file.name = fileLogger
+appender.file.fileName = ${logPath}
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+# Root logger level
+rootLogger.level = info
+
+# Root logger referring to console and file appenders
+rootLogger.appenderRef.stdout.ref = consoleLogger
+rootLogger.appenderRef.file.ref = fileLogger

integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java

@@ -97,6 +97,10 @@ protected void withLogConsumer(Consumer<OutputFrame> logConsumer) {
     container.withLogConsumer(logConsumer);
   }
 
+  protected void withStartupTimeout(Duration duration) {
+    container.withStartupTimeout(duration);
+  }
+
   // This method is used to get the expose port number of the container.
   public Integer getMappedPort(int exposedPort) {
     return container.getMappedPort(exposedPort);

integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java

@@ -36,6 +36,8 @@ public class ContainerSuite implements Closeable {
   private static TrinoContainer trinoContainer;
   private static TrinoITContainers trinoITContainers;
 
+  private static DorisContainer dorisContainer;
+
   protected static final CloseableGroup closer = CloseableGroup.create();
 
   private ContainerSuite() {
@@ -123,6 +125,17 @@ public void startTrinoContainer(
     trinoContainer.start();
   }
 
+  public void startDorisContainer() {
+    if (dorisContainer != null) {
+      return;
+    }
+    // Start Doris container
+    DorisContainer.Builder dorisBuilder =
+        DorisContainer.builder().withHostName("gravitino-ci-doris").withNetwork(network);
+    dorisContainer = closer.register(dorisBuilder.build());
+    dorisContainer.start();
+  }
+
   public TrinoContainer getTrinoContainer() {
     return trinoContainer;
   }
@@ -139,6 +152,10 @@ public HiveContainer getHiveContainer() {
     return hiveContainer;
   }
 
+  public DorisContainer getDorisContainer() {
+    return dorisContainer;
+  }
+
   // Let containers assign addresses in a fixed subnet to avoid `mac-docker-connector` needing to
   // refresh the configuration
   private static Network createDockerNetwork() {

integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+package com.datastrato.gravitino.integration.test.container;
+
+import static java.lang.String.format;
+
+import com.google.common.collect.ImmutableSet;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import org.rnorth.ducttape.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.Network;
+
+public class DorisContainer extends BaseContainer {
+  public static final Logger LOG = LoggerFactory.getLogger(DorisContainer.class);
+
+  public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_DORIS_DOCKER_IMAGE");
+  public static final String HOST_NAME = "gravitino-ci-doris";
+  public static final String USER_NAME = "root";
+  public static final String PASSWORD = "root";
+  public static final int FE_HTTP_PORT = 8030;
+  public static final int FE_MYSQL_PORT = 9030;
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  protected DorisContainer(
+      String image,
+      String hostName,
+      Set<Integer> ports,
+      Map<String, String> extraHosts,
+      Map<String, String> filesToMount,
+      Map<String, String> envVars,
+      Optional<Network> network) {
+    super(image, hostName, ports, extraHosts, filesToMount, envVars, network);
+  }
+
+  @Override
+  protected void setupContainer() {
+    super.setupContainer();
+    withLogConsumer(new PrintingContainerLog(format("%-14s| ", "DorisContainer")));
+    withStartupTimeout(Duration.ofMinutes(5));
+  }
+
+  @Override
+  public void start() {
+    super.start();
+    Preconditions.check("Doris container startup failed!", checkContainerStatus(5));
+    Preconditions.check("Doris container password change failed!", changePassword());
+  }
+
+  @Override
+  protected boolean checkContainerStatus(int retryLimit) {
+    int nRetry = 0;
+
+    String dorisJdbcUrl = format("jdbc:mysql://%s:%d/", getContainerIpAddress(), FE_MYSQL_PORT);
+    LOG.info("Doris url is " + dorisJdbcUrl);
+
+    while (nRetry++ < retryLimit) {
+      try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, "");
+          Statement statement = connection.createStatement()) {
+
+        // execute `SHOW PROC '/backends';` to check if backends is ready
+        String query = "SHOW PROC '/backends';";
+        try (ResultSet resultSet = statement.executeQuery(query)) {
+          while (resultSet.next()) {
+            String alive = resultSet.getString("Alive");
+            if (alive.equalsIgnoreCase("true")) {
+              LOG.info("Doris container startup success!");
+              return true;
+            }
+          }
+        }
+
+        LOG.info("Doris container is not ready yet!");
+        Thread.sleep(5000);
+      } catch (Exception e) {
+        LOG.error(e.getMessage(), e);
+      }
+    }
+
+    return false;
+  }
+
+  private boolean changePassword() {
+    String dorisJdbcUrl = format("jdbc:mysql://%s:%d/", getContainerIpAddress(), FE_MYSQL_PORT);
+
+    // change password for root user, Gravitino API must set password in catalog properties
+    try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, "");
+        Statement statement = connection.createStatement()) {
+
+      String query = String.format("SET PASSWORD FOR '%s' = PASSWORD('%s');", USER_NAME, PASSWORD);
+      statement.execute(query);
+      LOG.info("Doris container password has been changed");
+      return true;
+    } catch (Exception e) {
+      LOG.error(e.getMessage(), e);
+    }
+
+    return false;
+  }
+
+  public static class Builder
+      extends BaseContainer.Builder<DorisContainer.Builder, DorisContainer> {
+    private Builder() {
+      this.image = DEFAULT_IMAGE;
+      this.hostName = HOST_NAME;
+      this.exposePorts = ImmutableSet.of(FE_HTTP_PORT, FE_MYSQL_PORT);
+    }
+
+    @Override
+    public DorisContainer build() {
+      return new DorisContainer(
+          image, hostName, exposePorts, extraHosts, filesToMount, envVars, network);
+    }
+  }
+}

integration-test/build.gradle.kts

@@ -146,6 +146,7 @@ tasks.test {
 
     dependsOn(":trino-connector:jar")
     dependsOn(":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars")
+    dependsOn(":catalogs:catalog-jdbc-doris:jar", ":catalogs:catalog-jdbc-doris:runtimeJars")
     dependsOn(":catalogs:catalog-jdbc-mysql:jar", ":catalogs:catalog-jdbc-mysql:runtimeJars")
     dependsOn(":catalogs:catalog-jdbc-postgresql:jar", ":catalogs:catalog-jdbc-postgresql:runtimeJars")
     dependsOn(":catalogs:catalog-hadoop:jar", ":catalogs:catalog-hadoop:runtimeJars")