diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 69392503485..68c616f8ec9 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -58,6 +58,7 @@ jobs: architecture: [linux/amd64] java-version: [ 8, 11, 17 ] test-mode: [ embedded, deploy ] + backend: [ jdbcBackend, kvBackend] env: PLATFORM: ${{ matrix.architecture }} steps: @@ -87,7 +88,7 @@ jobs: - name: Backend Integration Test id: integrationTest run: | - ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs + ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/build.gradle.kts b/build.gradle.kts index f3d90d3b2b9..4ddf2ffac4d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -158,6 +158,9 @@ allprojects { // Change poll image pause time from 30s to 60s param.environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60") + if (project.hasProperty("jdbcBackend")) { + param.environment("jdbcBackend", "true") + } val testMode = project.properties["testMode"] as? String ?: "embedded" param.systemProperty("gravitino.log.path", project.buildDir.path + "/${project.name}-integration-test.log") diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index dbaf8210173..b29d0ebc30b 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -36,9 +36,11 @@ dependencies { testImplementation(libs.bundles.log4j) testImplementation(libs.mockito.core) + testImplementation(libs.mysql.driver) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.mysql) testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index ef761ece212..0603ba58519 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -92,6 +92,7 @@ dependencies { } testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) + testImplementation(libs.mysql.driver) testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") { exclude("org.apache.hadoop") @@ -106,6 +107,7 @@ dependencies { } testImplementation(libs.slf4j.api) testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.mysql) testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java index d46107305a4..f98a82a4b33 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java @@ -12,6 +12,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.SupportsMetalakes; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; @@ -121,7 +122,7 @@ public BaseMetalake createMetalake( try { store.put(metalake, false /* overwritten */); return metalake; - } catch (EntityAlreadyExistsException e) { + } catch (EntityAlreadyExistsException | AlreadyExistsException e) { LOG.warn("Metalake {} already exists", ident, e); throw new MetalakeAlreadyExistsException("Metalake %s already exists", ident); } catch (IOException ioe) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java index 009cdcb20e5..7f36478cf0a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java @@ -120,7 +120,7 @@ SchemaPO selectSchemaMetaByCatalogIdAndName( + " AND schema_name = #{oldSchemaMeta.schemaName}" + " AND metalake_id = #{oldSchemaMeta.metalakeId}" + " AND catalog_id = #{oldSchemaMeta.catalogId}" - + " AND schema_comment = #{oldSchemaMeta.schemaComment}" + + " AND (schema_comment IS NULL OR schema_comment = #{oldSchemaMeta.schemaComment})" + " AND properties = #{oldSchemaMeta.properties}" + " AND audit_info = #{oldSchemaMeta.auditInfo}" + " AND current_version = #{oldSchemaMeta.currentVersion}" diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index 9b996df9334..dcb85a6c60c 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -27,6 +27,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -34,12 +37,14 @@ import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MySQLContainer; @ExtendWith(PrintFuncNameExtension.class) public class AbstractIT { @@ -60,6 +65,13 @@ public class AbstractIT { protected static boolean ignoreIcebergRestService = true; + private static final String MYSQL_DOCKER_IMAGE_VERSION = "mysql:8.0"; + private static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.26/mysql-connector-java-8.0.26.jar"; + + private static final String META_DATA = "metadata"; + private static MySQLContainer MYSQL_CONTAINER; + protected static String serverUri; public static int getGravitinoServerPort() { @@ -98,6 +110,47 @@ private static void recoverGravitinoServerConfig() throws IOException { Files.move(tmpPath, configPath); } + protected static void downLoadMySQLDriver(String relativeDeployLibsPath) throws IOException { + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + java.nio.file.Path tmpPath = Paths.get(gravitinoHome, relativeDeployLibsPath); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } + } + + private static void setMySQLBackend() { + String mysqlUrl = MYSQL_CONTAINER.getJdbcUrl(); + customConfigs.put(Configs.ENTITY_STORE_KEY, "relational"); + customConfigs.put(Configs.ENTITY_RELATIONAL_STORE_KEY, "JDBCBackend"); + customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY, mysqlUrl); + customConfigs.put( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY, "com.mysql.cj.jdbc.Driver"); + customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY, "root"); + customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY, "root"); + + LOG.info("MySQL URL: {}", mysqlUrl); + // Connect to the mysql docker and create a databases + try (Connection connection = + DriverManager.getConnection( + StringUtils.substring(mysqlUrl, 0, mysqlUrl.lastIndexOf("/")), "root", "root"); + final Statement statement = connection.createStatement()) { + statement.execute("drop database if exists " + META_DATA); + statement.execute("create database " + META_DATA); + String gravitinoHome = System.getenv("GRAVITINO_ROOT_DIR"); + String mysqlContent = + FileUtils.readFileToString( + new File(gravitinoHome + "/core/src/main/resources/mysql/mysql_init.sql"), "UTF-8"); + String[] initMySQLBackendSqls = mysqlContent.split(";"); + initMySQLBackendSqls = ArrayUtils.addFirst(initMySQLBackendSqls, "use " + META_DATA + ";"); + for (String sql : initMySQLBackendSqls) { + statement.execute(sql); + } + } catch (Exception e) { + LOG.error("Failed to create database in mysql", e); + throw new RuntimeException(e); + } + } + @BeforeAll public static void startIntegrationTest() throws Exception { testMode = @@ -107,6 +160,18 @@ public static void startIntegrationTest() throws Exception { LOG.info("Running Gravitino Server in {} mode", testMode); + if ("true".equals(System.getenv("jdbcBackend"))) { + // Start MySQL docker instance. + MYSQL_CONTAINER = + new MySQLContainer<>(MYSQL_DOCKER_IMAGE_VERSION) + .withDatabaseName(META_DATA) + .withUsername("root") + .withPassword("root"); + MYSQL_CONTAINER.start(); + + setMySQLBackend(); + } + serverConfig = new ServerConfig(); if (testMode != null && testMode.equals(ITUtils.EMBEDDED_TEST_MODE)) { MiniGravitinoContext context = @@ -117,6 +182,7 @@ public static void startIntegrationTest() throws Exception { } else { rewriteGravitinoServerConfig(); serverConfig.loadFromFile(GravitinoServer.CONF_FILE); + downLoadMySQLDriver("/libs"); try { FileUtils.deleteDirectory( FileUtils.getFile(serverConfig.get(ENTRY_KV_ROCKSDB_BACKEND_PATH))); @@ -168,6 +234,10 @@ public static void stopIntegrationTest() throws IOException, InterruptedExceptio } customConfigs.clear(); LOG.info("Tearing down Gravitino Server"); + + if (MYSQL_CONTAINER != null) { + MYSQL_CONTAINER.stop(); + } } public static GravitinoAdminClient getGravitinoClient() {