Fix

apache · Jul 1, 2024 · 6e131ce · 6e131ce
1 parent 445792f
commit 6e131ce
Show file tree

Hide file tree

Showing 17 changed files with 42 additions and 245 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -61,7 +61,7 @@ jobs:
         architecture: [linux/amd64]
         java-version: [ 8, 11, 17 ]
         test-mode: [ embedded, deploy ]
-        backend: [ jdbcBackend, kvBackend]
+        backend: [ mysql, h2]
     env:
       PLATFORM: ${{ matrix.architecture }}
     steps:

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -164,9 +164,8 @@ allprojects {
 
       // Change poll image pause time from 30s to 60s
       param.environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60")
-      if (project.hasProperty("jdbcBackend")) {
-        param.environment("jdbcBackend", "true")
-      }
+      val jdbcDatabase = project.properties["jdbcBackend"] as? String ?: "h2"
+      param.environment("jdbcBackend", jdbcDatabase)
 
       val testMode = project.properties["testMode"] as? String ?: "embedded"
       param.systemProperty("gravitino.log.path", project.buildDir.path + "/${project.name}-integration-test.log")

diff --git a/...op/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/...op/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
@@ -38,6 +38,7 @@
 import com.datastrato.gravitino.utils.NameIdentifierUtil;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.Arrays;
@@ -108,6 +109,8 @@ public static void setUp() {
     Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE);
     Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto");
     Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH);
+    File f = FileUtils.getFile(ROCKS_DB_STORE_PATH);
+    f.deleteOnExit();
 
     Assertions.assertEquals(ROCKS_DB_STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH));
     Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
@@ -122,7 +125,6 @@ public static void setUp() {
   @AfterAll
   public static void tearDown() throws IOException {
     store.close();
-    FileUtils.deleteDirectory(FileUtils.getFile(ROCKS_DB_STORE_PATH));
     new Path(TEST_ROOT_PATH)
         .getFileSystem(new Configuration())
         .delete(new Path(TEST_ROOT_PATH), true);

diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java
@@ -16,7 +16,7 @@ public class Configs {
 
   private Configs() {}
 
-  public static final String DEFAULT_ENTITY_STORE = "kv";
+  public static final String KV_STORE_KEY = "kv";
   public static final String RELATIONAL_ENTITY_STORE = "relational";
   public static final String ENTITY_STORE_KEY = "gravitino.entity.store";
 

diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java
@@ -4,6 +4,8 @@
  */
 package com.datastrato.gravitino;
 
+import static com.datastrato.gravitino.Configs.KV_STORE_KEY;
+
 import com.datastrato.gravitino.storage.kv.KvEntityStore;
 import com.datastrato.gravitino.storage.relational.RelationalEntityStore;
 import com.google.common.collect.ImmutableMap;
@@ -22,7 +24,7 @@ public class EntityStoreFactory {
   // doesn't need to specify the full qualified class name when creating an EntityStore.
   public static final ImmutableMap<String, String> ENTITY_STORES =
       ImmutableMap.of(
-          Configs.DEFAULT_ENTITY_STORE,
+          KV_STORE_KEY,
           KvEntityStore.class.getCanonicalName(),
           Configs.RELATIONAL_ENTITY_STORE,
           RelationalEntityStore.class.getCanonicalName());
@@ -40,6 +42,11 @@ public static EntityStore createEntityStore(Config config) {
     String name = config.get(Configs.ENTITY_STORE);
     String className = ENTITY_STORES.getOrDefault(name, name);
 
+    if (KV_STORE_KEY.equals(name)) {
+      throw new UnsupportedOperationException(
+          "KvEntityStore is not supported since this version. Please use RelationalEntityStore instead.");
+    }
+
     try {
       return (EntityStore) Class.forName(className).getDeclaredConstructor().newInstance();
     } catch (Exception e) {

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java
@@ -49,32 +49,30 @@
 import com.datastrato.gravitino.meta.TableEntity;
 import com.datastrato.gravitino.meta.TopicEntity;
 import com.datastrato.gravitino.meta.UserEntity;
-import com.datastrato.gravitino.storage.relational.RelationalEntityStore;
 import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
 import java.util.UUID;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.ibatis.session.SqlSession;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.Mockito;
 
+@Disabled
 public class TestEntityStorage {
   public static final String KV_STORE_PATH =
       "/tmp/gravitino_kv_entityStore_" + UUID.randomUUID().toString().replace("-", "");
@@ -84,12 +82,12 @@ public class TestEntityStorage {
   private static final String DB_DIR = JDBC_STORE_PATH + "/testdb";
 
   static Object[] storageProvider() {
-    return new Object[] {Configs.DEFAULT_ENTITY_STORE, Configs.RELATIONAL_ENTITY_STORE};
+    return new Object[] {Configs.RELATIONAL_ENTITY_STORE};
   }
 
   private void init(String type, Config config) {
     Preconditions.checkArgument(StringUtils.isNotBlank(type));
-    if (type.equals(Configs.DEFAULT_ENTITY_STORE)) {
+    if (type.equals(Configs.KV_STORE_KEY)) {
       try {
         FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH));
       } catch (Exception e) {
@@ -131,28 +129,9 @@ private void init(String type, Config config) {
     }
   }
 
-  private void prepareJdbcTable() {
-    // Read the ddl sql to create table
-    String scriptPath = "h2/schema-h2.sql";
-    try (SqlSession sqlSession =
-            SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true);
-        Connection connection = sqlSession.getConnection();
-        Statement statement = connection.createStatement()) {
-      StringBuilder ddlBuilder = new StringBuilder();
-      IOUtils.readLines(
-              Objects.requireNonNull(
-                  this.getClass().getClassLoader().getResourceAsStream(scriptPath)),
-              StandardCharsets.UTF_8)
-          .forEach(line -> ddlBuilder.append(line).append("\n"));
-      statement.execute(ddlBuilder.toString());
-    } catch (Exception e) {
-      throw new IllegalStateException("Create tables failed", e);
-    }
-  }
-
   private void destroy(String type) {
     Preconditions.checkArgument(StringUtils.isNotBlank(type));
-    if (type.equals(Configs.DEFAULT_ENTITY_STORE)) {
+    if (type.equals(Configs.KV_STORE_KEY)) {
       try {
         FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH));
       } catch (Exception e) {
@@ -2121,9 +2100,9 @@ void testOptimizedDeleteForKv(String type) throws IOException {
 
     try (EntityStore store = EntityStoreFactory.createEntityStore(config)) {
       store.initialize(config);
-      if (store instanceof RelationalEntityStore) {
-        prepareJdbcTable();
-      }
+      //      if (store instanceof RelationalEntityStore) {
+      //        prepareJdbcTable();
+      //      }
 
       BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo);
       CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"), "catalog", auditInfo);

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java
@@ -32,6 +32,7 @@
 import java.util.stream.Stream;
 import org.apache.commons.lang3.tuple.Pair;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
@@ -41,6 +42,7 @@
 import org.mockito.Mockito;
 
 @TestInstance(Lifecycle.PER_CLASS)
+@Disabled
 public class TestEntityKeyEncoding {
   private Config getConfig() throws IOException {
     File baseDir = new File(System.getProperty("java.io.tmpdir"));

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java
@@ -49,6 +49,7 @@
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
+@Disabled
 public class TestKvEntityStorage extends TestEntityStorage {
   @BeforeEach
   @AfterEach

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java
@@ -45,10 +45,12 @@
 import java.util.List;
 import org.apache.commons.lang3.tuple.Pair;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
 @SuppressWarnings("DefaultCharset")
+@Disabled
 class TestKvGarbageCollector {
   public Config getConfig() throws IOException {
     Config config = Mockito.mock(Config.class);

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java
@@ -23,9 +23,11 @@
 import java.lang.reflect.Field;
 import java.nio.file.Files;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
+@Disabled
 public class TestKvNameMappingService {
   private Config getConfig() throws IOException {
     File baseDir = new File(System.getProperty("java.io.tmpdir"));

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java
@@ -23,9 +23,11 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
+@Disabled
 class TestStorageVersion {
 
   @Test

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java
@@ -83,6 +83,7 @@ public class TestJDBCBackend {
   @BeforeAll
   public static void setup() {
     File dir = new File(DB_DIR);
+    dir.deleteOnExit();
     if (dir.exists() || !dir.isDirectory()) {
       dir.delete();
     }

diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java
@@ -24,6 +24,7 @@
 import java.sql.SQLException;
 import java.util.UUID;
 import org.apache.commons.dbcp2.BasicDataSource;
+import org.apache.commons.io.FileUtils;
 import org.apache.ibatis.session.SqlSession;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
@@ -71,8 +72,9 @@ public void cleanUp() {
   public static void tearDown() throws IOException {
     File dir = new File(DB_DIR);
     if (dir.exists()) {
-      dir.delete();
+      FileUtils.deleteDirectory(FileUtils.getFile(DB_DIR));
     }
+
     SqlSessionFactoryHelper.getInstance().close();
   }