From 4cbdeb33824cb64488347955847436e4daa02775 Mon Sep 17 00:00:00 2001 From: Zhou Kang Date: Tue, 19 Mar 2024 21:36:03 +0800 Subject: [PATCH 1/4] add Doris container implement for integration-test --- catalogs/catalog-jdbc-doris/build.gradle.kts | 17 +++ .../integration/test/TestDorisAbstractIT.java | 62 ++++++++ .../test/TestDorisDatabaseOperations.java | 29 ++++ .../src/test/resources/log4j2.properties | 33 +++++ .../test/container/BaseContainer.java | 4 + .../test/container/ContainerSuite.java | 17 +++ .../test/container/DorisContainer.java | 132 ++++++++++++++++++ integration-test/build.gradle.kts | 1 + 8 files changed, 295 insertions(+) create mode 100644 catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisAbstractIT.java create mode 100644 catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisDatabaseOperations.java create mode 100644 catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties create mode 100644 integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index 77040ed44a9..21ea928c53d 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -22,6 +22,19 @@ dependencies { implementation(libs.guava) implementation(libs.jsqlparser) implementation(libs.slf4j.api) + + testImplementation(project(":catalogs:catalog-jdbc-common", "testArtifacts")) + testImplementation(project(":integration-test-common", "testArtifacts")) + + testImplementation(libs.commons.lang3) + testImplementation(libs.guava) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mysql.driver) + testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.mysql) + + testRuntimeOnly(libs.junit.jupiter.engine) } tasks { @@ -65,6 +78,10 @@ tasks.test { } else { dependsOn(tasks.jar) + doFirst { + environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.0") + } + val init = project.extra.get("initIntegrationTest") as (Test) -> Unit init(this) } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisAbstractIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisAbstractIT.java new file mode 100644 index 00000000000..a169513e0bd --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisAbstractIT.java @@ -0,0 +1,62 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.integration.test; + +import com.datastrato.gravitino.catalog.doris.converter.DorisColumnDefaultValueConverter; +import com.datastrato.gravitino.catalog.doris.converter.DorisExceptionConverter; +import com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter; +import com.datastrato.gravitino.catalog.doris.operation.DorisDatabaseOperations; +import com.datastrato.gravitino.catalog.doris.operation.DorisTableOperations; +import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; +import com.datastrato.gravitino.catalog.jdbc.integration.test.TestJdbcAbstractIT; +import com.datastrato.gravitino.catalog.jdbc.utils.DataSourceUtils; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.DorisContainer; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeAll; + +public class TestDorisAbstractIT extends TestJdbcAbstractIT { + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; + + @BeforeAll + public static void startup() { + containerSuite.startDorisContainer(); + + DataSource dataSource = DataSourceUtils.createDataSource(getDorisCatalogProperties()); + + DATABASE_OPERATIONS = new DorisDatabaseOperations(); + TABLE_OPERATIONS = new DorisTableOperations(); + JDBC_EXCEPTION_CONVERTER = new DorisExceptionConverter(); + DATABASE_OPERATIONS.initialize(dataSource, JDBC_EXCEPTION_CONVERTER, Collections.emptyMap()); + TABLE_OPERATIONS.initialize( + dataSource, + JDBC_EXCEPTION_CONVERTER, + new DorisTypeConverter(), + new DorisColumnDefaultValueConverter(), + Collections.emptyMap()); + } + + private static Map getDorisCatalogProperties() { + Map catalogProperties = Maps.newHashMap(); + + DorisContainer dorisContainer = containerSuite.getDorisContainer(); + + String jdbcUrl = + String.format( + "jdbc:mysql://%s:%d/", + dorisContainer.getContainerIpAddress(), DorisContainer.FE_MYSQL_PORT); + + catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl); + catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), DRIVER_CLASS_NAME); + catalogProperties.put(JdbcConfig.USERNAME.getKey(), DorisContainer.USER_NAME); + catalogProperties.put(JdbcConfig.PASSWORD.getKey(), DorisContainer.PASSWORD); + + return catalogProperties; + } +} diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisDatabaseOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisDatabaseOperations.java new file mode 100644 index 00000000000..ea896675bfe --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/TestDorisDatabaseOperations.java @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.integration.test; + +import com.datastrato.gravitino.utils.RandomNameUtils; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("gravitino-docker-it") +public class TestDorisDatabaseOperations extends TestDorisAbstractIT { + + @Test + public void testBaseOperationDatabase() { + String databaseName = RandomNameUtils.genRandomName("ct_db"); + Map properties = new HashMap<>(); + String databaseComment = "test_comment"; + + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> { + DATABASE_OPERATIONS.create(databaseName, databaseComment, properties); + }); + } +} diff --git a/catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties b/catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..7db40f053da --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/test/resources/log4j2.properties @@ -0,0 +1,33 @@ +# +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-catalog-jdbc-doris/build/doris-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java index 331efae498e..b0fb3973c9e 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java @@ -97,6 +97,10 @@ protected void withLogConsumer(Consumer logConsumer) { container.withLogConsumer(logConsumer); } + protected void withStartupTimeout(Duration duration) { + container.withStartupTimeout(duration); + } + // This method is used to get the expose port number of the container. public Integer getMappedPort(int exposedPort) { return container.getMappedPort(exposedPort); diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index c02095dfb5b..e2344ab0928 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -36,6 +36,8 @@ public class ContainerSuite implements Closeable { private static TrinoContainer trinoContainer; private static TrinoITContainers trinoITContainers; + private static DorisContainer dorisContainer; + protected static final CloseableGroup closer = CloseableGroup.create(); private ContainerSuite() { @@ -123,6 +125,17 @@ public void startTrinoContainer( trinoContainer.start(); } + public void startDorisContainer() { + if (dorisContainer != null) { + return; + } + // Start Doris container + DorisContainer.Builder dorisBuilder = + DorisContainer.builder().withHostName("gravitino-ci-doris").withNetwork(network); + dorisContainer = closer.register(dorisBuilder.build()); + dorisContainer.start(); + } + public TrinoContainer getTrinoContainer() { return trinoContainer; } @@ -139,6 +152,10 @@ public HiveContainer getHiveContainer() { return hiveContainer; } + public DorisContainer getDorisContainer() { + return dorisContainer; + } + // Let containers assign addresses in a fixed subnet to avoid `mac-docker-connector` needing to // refresh the configuration private static Network createDockerNetwork() { diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java new file mode 100644 index 00000000000..76d971fefd0 --- /dev/null +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java @@ -0,0 +1,132 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.integration.test.container; + +import static java.lang.String.format; + +import com.google.common.collect.ImmutableSet; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Duration; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.rnorth.ducttape.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Network; + +public class DorisContainer extends BaseContainer { + public static final Logger LOG = LoggerFactory.getLogger(DorisContainer.class); + + public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_DORIS_DOCKER_IMAGE"); + public static final String HOST_NAME = "gravitino-ci-doris"; + public static final String USER_NAME = "root"; + public static final String PASSWORD = "root"; + public static final int FE_HTTP_PORT = 8030; + public static final int FE_MYSQL_PORT = 9030; + + public static Builder builder() { + return new Builder(); + } + + protected DorisContainer( + String image, + String hostName, + Set ports, + Map extraHosts, + Map filesToMount, + Map envVars, + Optional network) { + super(image, hostName, ports, extraHosts, filesToMount, envVars, network); + } + + @Override + protected void setupContainer() { + super.setupContainer(); + withLogConsumer(new PrintingContainerLog(format("%-14s| ", "DorisContainer"))); + withStartupTimeout(Duration.ofMinutes(5)); + } + + @Override + public void start() { + super.start(); + Preconditions.check("Doris container startup failed!", checkContainerStatus(5)); + Preconditions.check("Doris container password change failed!", changePassword()); + } + + @Override + protected boolean checkContainerStatus(int retryLimit) { + int nRetry = 0; + boolean isDorisContainerReady = false; + + String dorisJdbcUrl = format("jdbc:mysql://%s:%d/", getContainerIpAddress(), FE_MYSQL_PORT); + LOG.info("Doris url is " + dorisJdbcUrl); + + while (nRetry++ < retryLimit) { + try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, "")) { + + // execute `SHOW PROC '/backends';` to check if backends is ready + String query = "SHOW PROC '/backends';"; + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(query); + while (resultSet.next()) { + String alive = resultSet.getString("Alive"); + if (alive.equalsIgnoreCase("true")) { + LOG.info("Doris container startup success!"); + isDorisContainerReady = true; + break; + } + } + + if (isDorisContainerReady) { + break; + } + LOG.info("Doris container is not ready yet!"); + Thread.sleep(5000); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + } + + return isDorisContainerReady; + } + + private boolean changePassword() { + boolean result = false; + String dorisJdbcUrl = format("jdbc:mysql://%s:%d/", getContainerIpAddress(), FE_MYSQL_PORT); + + // change password for root user, Gravitino API must set password in catalog properties + try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, "")) { + + String query = String.format("SET PASSWORD FOR '%s' = PASSWORD('%s');", USER_NAME, PASSWORD); + Statement statement = connection.createStatement(); + statement.execute(query); + LOG.info("Doris container password has been changed"); + result = true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + } + + return result; + } + + public static class Builder + extends BaseContainer.Builder { + private Builder() { + this.image = DEFAULT_IMAGE; + this.hostName = HOST_NAME; + this.exposePorts = ImmutableSet.of(FE_HTTP_PORT, FE_MYSQL_PORT); + } + + @Override + public DorisContainer build() { + return new DorisContainer( + image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); + } + } +} diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index cccae579103..ffdc62e653b 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -146,6 +146,7 @@ tasks.test { dependsOn(":trino-connector:jar") dependsOn(":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars") + dependsOn(":catalogs:catalog-jdbc-doris:jar", ":catalogs:catalog-jdbc-doris:runtimeJars") dependsOn(":catalogs:catalog-jdbc-mysql:jar", ":catalogs:catalog-jdbc-mysql:runtimeJars") dependsOn(":catalogs:catalog-jdbc-postgresql:jar", ":catalogs:catalog-jdbc-postgresql:runtimeJars") dependsOn(":catalogs:catalog-hadoop:jar", ":catalogs:catalog-hadoop:runtimeJars") From ef2feb8052e45399a9ccf62b6c6ad8aedecd7eb6 Mon Sep 17 00:00:00 2001 From: Zhou Kang Date: Tue, 26 Mar 2024 09:31:27 +0800 Subject: [PATCH 2/4] update timeout-minutes to 60 --- .github/workflows/backend-integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 795e26c62ad..c1107db62ed 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -55,7 +55,7 @@ jobs: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: architecture: [linux/amd64] From 70eeebb7a6f6eb26418c49d24cebe0325febe0e4 Mon Sep 17 00:00:00 2001 From: Zhou Kang Date: Fri, 22 Mar 2024 13:17:01 +0800 Subject: [PATCH 3/4] update docker version --- catalogs/catalog-jdbc-doris/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index 21ea928c53d..9f49e804f2d 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -79,7 +79,7 @@ tasks.test { dependsOn(tasks.jar) doFirst { - environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.0") + environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.1") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit From 1d20e0d22ce4844f518cc801a1c3479a94997b72 Mon Sep 17 00:00:00 2001 From: Zhou Kang Date: Thu, 28 Mar 2024 19:32:38 +0800 Subject: [PATCH 4/4] minor update --- .../test/container/DorisContainer.java | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java index 76d971fefd0..ad0790dcdeb 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java @@ -62,30 +62,26 @@ public void start() { @Override protected boolean checkContainerStatus(int retryLimit) { int nRetry = 0; - boolean isDorisContainerReady = false; String dorisJdbcUrl = format("jdbc:mysql://%s:%d/", getContainerIpAddress(), FE_MYSQL_PORT); LOG.info("Doris url is " + dorisJdbcUrl); while (nRetry++ < retryLimit) { - try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, "")) { + try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, ""); + Statement statement = connection.createStatement()) { // execute `SHOW PROC '/backends';` to check if backends is ready String query = "SHOW PROC '/backends';"; - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(query); - while (resultSet.next()) { - String alive = resultSet.getString("Alive"); - if (alive.equalsIgnoreCase("true")) { - LOG.info("Doris container startup success!"); - isDorisContainerReady = true; - break; + try (ResultSet resultSet = statement.executeQuery(query)) { + while (resultSet.next()) { + String alive = resultSet.getString("Alive"); + if (alive.equalsIgnoreCase("true")) { + LOG.info("Doris container startup success!"); + return true; + } } } - if (isDorisContainerReady) { - break; - } LOG.info("Doris container is not ready yet!"); Thread.sleep(5000); } catch (Exception e) { @@ -93,26 +89,25 @@ protected boolean checkContainerStatus(int retryLimit) { } } - return isDorisContainerReady; + return false; } private boolean changePassword() { - boolean result = false; String dorisJdbcUrl = format("jdbc:mysql://%s:%d/", getContainerIpAddress(), FE_MYSQL_PORT); // change password for root user, Gravitino API must set password in catalog properties - try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, "")) { + try (Connection connection = DriverManager.getConnection(dorisJdbcUrl, USER_NAME, ""); + Statement statement = connection.createStatement()) { String query = String.format("SET PASSWORD FOR '%s' = PASSWORD('%s');", USER_NAME, PASSWORD); - Statement statement = connection.createStatement(); statement.execute(query); LOG.info("Doris container password has been changed"); - result = true; + return true; } catch (Exception e) { LOG.error(e.getMessage(), e); } - return result; + return false; } public static class Builder