From d93ce3d6aac337616871de5b4b6705e18de78222 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 3 Apr 2024 09:27:43 +0800 Subject: [PATCH] Revert "[#2678] improvement(test): Add e2e test for jdbc backend (#2686)" This reverts commit 2bfefb9b46d0abfef57f025f5258bb82eb785d9d. --- .../workflows/backend-integration-test.yml | 3 +- build.gradle.kts | 3 - catalogs/catalog-hadoop/build.gradle.kts | 2 - catalogs/catalog-hive/build.gradle.kts | 2 - .../gravitino/metalake/MetalakeManager.java | 3 +- .../relational/mapper/SchemaMetaMapper.java | 2 +- .../integration/test/util/AbstractIT.java | 70 ------------------- 7 files changed, 3 insertions(+), 82 deletions(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 68c616f8ec9..69392503485 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -58,7 +58,6 @@ jobs: architecture: [linux/amd64] java-version: [ 8, 11, 17 ] test-mode: [ embedded, deploy ] - backend: [ jdbcBackend, kvBackend] env: PLATFORM: ${{ matrix.architecture }} steps: @@ -88,7 +87,7 @@ jobs: - name: Backend Integration Test id: integrationTest run: | - ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs -P${{ matrix.backend }} + ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipWebITs - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/build.gradle.kts b/build.gradle.kts index 4ddf2ffac4d..f3d90d3b2b9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -158,9 +158,6 @@ allprojects { // Change poll image pause time from 30s to 60s param.environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60") - if (project.hasProperty("jdbcBackend")) { - param.environment("jdbcBackend", "true") - } val testMode = project.properties["testMode"] as? String ?: "embedded" param.systemProperty("gravitino.log.path", project.buildDir.path + "/${project.name}-integration-test.log") diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index b29d0ebc30b..dbaf8210173 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -36,11 +36,9 @@ dependencies { testImplementation(libs.bundles.log4j) testImplementation(libs.mockito.core) - testImplementation(libs.mysql.driver) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.mysql) testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 0603ba58519..ef761ece212 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -92,7 +92,6 @@ dependencies { } testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) - testImplementation(libs.mysql.driver) testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") { exclude("org.apache.hadoop") @@ -107,7 +106,6 @@ dependencies { } testImplementation(libs.slf4j.api) testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.mysql) testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java index f98a82a4b33..d46107305a4 100644 --- a/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/com/datastrato/gravitino/metalake/MetalakeManager.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.SupportsMetalakes; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; @@ -122,7 +121,7 @@ public BaseMetalake createMetalake( try { store.put(metalake, false /* overwritten */); return metalake; - } catch (EntityAlreadyExistsException | AlreadyExistsException e) { + } catch (EntityAlreadyExistsException e) { LOG.warn("Metalake {} already exists", ident, e); throw new MetalakeAlreadyExistsException("Metalake %s already exists", ident); } catch (IOException ioe) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java index 7f36478cf0a..009cdcb20e5 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java @@ -120,7 +120,7 @@ SchemaPO selectSchemaMetaByCatalogIdAndName( + " AND schema_name = #{oldSchemaMeta.schemaName}" + " AND metalake_id = #{oldSchemaMeta.metalakeId}" + " AND catalog_id = #{oldSchemaMeta.catalogId}" - + " AND (schema_comment IS NULL OR schema_comment = #{oldSchemaMeta.schemaComment})" + + " AND schema_comment = #{oldSchemaMeta.schemaComment}" + " AND properties = #{oldSchemaMeta.properties}" + " AND audit_info = #{oldSchemaMeta.auditInfo}" + " AND current_version = #{oldSchemaMeta.currentVersion}" diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index dcb85a6c60c..9b996df9334 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -27,9 +27,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -37,14 +34,12 @@ import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.MySQLContainer; @ExtendWith(PrintFuncNameExtension.class) public class AbstractIT { @@ -65,13 +60,6 @@ public class AbstractIT { protected static boolean ignoreIcebergRestService = true; - private static final String MYSQL_DOCKER_IMAGE_VERSION = "mysql:8.0"; - private static final String DOWNLOAD_JDBC_DRIVER_URL = - "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.26/mysql-connector-java-8.0.26.jar"; - - private static final String META_DATA = "metadata"; - private static MySQLContainer MYSQL_CONTAINER; - protected static String serverUri; public static int getGravitinoServerPort() { @@ -110,47 +98,6 @@ private static void recoverGravitinoServerConfig() throws IOException { Files.move(tmpPath, configPath); } - protected static void downLoadMySQLDriver(String relativeDeployLibsPath) throws IOException { - if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { - String gravitinoHome = System.getenv("GRAVITINO_HOME"); - java.nio.file.Path tmpPath = Paths.get(gravitinoHome, relativeDeployLibsPath); - JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); - } - } - - private static void setMySQLBackend() { - String mysqlUrl = MYSQL_CONTAINER.getJdbcUrl(); - customConfigs.put(Configs.ENTITY_STORE_KEY, "relational"); - customConfigs.put(Configs.ENTITY_RELATIONAL_STORE_KEY, "JDBCBackend"); - customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY, mysqlUrl); - customConfigs.put( - Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY, "com.mysql.cj.jdbc.Driver"); - customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY, "root"); - customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY, "root"); - - LOG.info("MySQL URL: {}", mysqlUrl); - // Connect to the mysql docker and create a databases - try (Connection connection = - DriverManager.getConnection( - StringUtils.substring(mysqlUrl, 0, mysqlUrl.lastIndexOf("/")), "root", "root"); - final Statement statement = connection.createStatement()) { - statement.execute("drop database if exists " + META_DATA); - statement.execute("create database " + META_DATA); - String gravitinoHome = System.getenv("GRAVITINO_ROOT_DIR"); - String mysqlContent = - FileUtils.readFileToString( - new File(gravitinoHome + "/core/src/main/resources/mysql/mysql_init.sql"), "UTF-8"); - String[] initMySQLBackendSqls = mysqlContent.split(";"); - initMySQLBackendSqls = ArrayUtils.addFirst(initMySQLBackendSqls, "use " + META_DATA + ";"); - for (String sql : initMySQLBackendSqls) { - statement.execute(sql); - } - } catch (Exception e) { - LOG.error("Failed to create database in mysql", e); - throw new RuntimeException(e); - } - } - @BeforeAll public static void startIntegrationTest() throws Exception { testMode = @@ -160,18 +107,6 @@ public static void startIntegrationTest() throws Exception { LOG.info("Running Gravitino Server in {} mode", testMode); - if ("true".equals(System.getenv("jdbcBackend"))) { - // Start MySQL docker instance. - MYSQL_CONTAINER = - new MySQLContainer<>(MYSQL_DOCKER_IMAGE_VERSION) - .withDatabaseName(META_DATA) - .withUsername("root") - .withPassword("root"); - MYSQL_CONTAINER.start(); - - setMySQLBackend(); - } - serverConfig = new ServerConfig(); if (testMode != null && testMode.equals(ITUtils.EMBEDDED_TEST_MODE)) { MiniGravitinoContext context = @@ -182,7 +117,6 @@ public static void startIntegrationTest() throws Exception { } else { rewriteGravitinoServerConfig(); serverConfig.loadFromFile(GravitinoServer.CONF_FILE); - downLoadMySQLDriver("/libs"); try { FileUtils.deleteDirectory( FileUtils.getFile(serverConfig.get(ENTRY_KV_ROCKSDB_BACKEND_PATH))); @@ -234,10 +168,6 @@ public static void stopIntegrationTest() throws IOException, InterruptedExceptio } customConfigs.clear(); LOG.info("Tearing down Gravitino Server"); - - if (MYSQL_CONTAINER != null) { - MYSQL_CONTAINER.stop(); - } } public static GravitinoAdminClient getGravitinoClient() {