[#5633] remove the key-value storage backend logic

apache · Dec 4, 2024 · a42fae7 · a42fae7
1 parent 054c6f7
commit a42fae7
Show file tree

Hide file tree

Showing 75 changed files with 56 additions and 8,126 deletions.
diff --git a/LICENSE.bin b/LICENSE.bin
@@ -384,7 +384,6 @@
 
    JSR305
    LevelDB JNI
-   RocksDB JNI
    JCraft
    Javolution
    ANTLR4

diff --git a/LICENSE.rest b/LICENSE.rest
@@ -314,7 +314,6 @@
    JSR305
    LevelDB JNI
    Protobuf
-   RocksDB JNI
    Stax2 API
    XMLenc
 

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -303,51 +303,49 @@ subprojects {
     }
   }
 
-  if (project.name != "meta") {
-    apply(plugin = "net.ltgt.errorprone")
-    dependencies {
-      errorprone("com.google.errorprone:error_prone_core:2.10.0")
-    }
-
-    tasks.withType<JavaCompile>().configureEach {
-      options.errorprone.isEnabled.set(true)
-      options.errorprone.disableWarningsInGeneratedCode.set(true)
-      options.errorprone.disable(
-        "AlmostJavadoc",
-        "CanonicalDuration",
-        "CheckReturnValue",
-        "ComparableType",
-        "ConstantOverflow",
-        "DoubleBraceInitialization",
-        "EqualsUnsafeCast",
-        "EmptyBlockTag",
-        "FutureReturnValueIgnored",
-        "InconsistentCapitalization",
-        "InconsistentHashCode",
-        "JavaTimeDefaultTimeZone",
-        "JdkObsolete",
-        "LockNotBeforeTry",
-        "MissingSummary",
-        "MissingOverride",
-        "MutableConstantField",
-        "NonOverridingEquals",
-        "ObjectEqualsForPrimitives",
-        "OperatorPrecedence",
-        "ReturnValueIgnored",
-        "SameNameButDifferent",
-        "StaticAssignmentInConstructor",
-        "StringSplitter",
-        "ThreadPriorityCheck",
-        "ThrowIfUncheckedKnownChecked",
-        "TypeParameterUnusedInFormals",
-        "UnicodeEscape",
-        "UnnecessaryParentheses",
-        "UnsafeReflectiveConstructionCast",
-        "UnusedMethod",
-        "VariableNameSameAsType",
-        "WaitNotInLoop"
-      )
-    }
+  apply(plugin = "net.ltgt.errorprone")
+  dependencies {
+    errorprone("com.google.errorprone:error_prone_core:2.10.0")
+  }
+
+  tasks.withType<JavaCompile>().configureEach {
+    options.errorprone.isEnabled.set(true)
+    options.errorprone.disableWarningsInGeneratedCode.set(true)
+    options.errorprone.disable(
+      "AlmostJavadoc",
+      "CanonicalDuration",
+      "CheckReturnValue",
+      "ComparableType",
+      "ConstantOverflow",
+      "DoubleBraceInitialization",
+      "EqualsUnsafeCast",
+      "EmptyBlockTag",
+      "FutureReturnValueIgnored",
+      "InconsistentCapitalization",
+      "InconsistentHashCode",
+      "JavaTimeDefaultTimeZone",
+      "JdkObsolete",
+      "LockNotBeforeTry",
+      "MissingSummary",
+      "MissingOverride",
+      "MutableConstantField",
+      "NonOverridingEquals",
+      "ObjectEqualsForPrimitives",
+      "OperatorPrecedence",
+      "ReturnValueIgnored",
+      "SameNameButDifferent",
+      "StaticAssignmentInConstructor",
+      "StringSplitter",
+      "ThreadPriorityCheck",
+      "ThrowIfUncheckedKnownChecked",
+      "TypeParameterUnusedInFormals",
+      "UnicodeEscape",
+      "UnnecessaryParentheses",
+      "UnsafeReflectiveConstructionCast",
+      "UnusedMethod",
+      "VariableNameSameAsType",
+      "WaitNotInLoop"
+    )
   }
 
   tasks.withType<Javadoc> {

diff --git a/...hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/...hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java
@@ -25,7 +25,6 @@
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_STORE;
-import static org.apache.gravitino.Configs.ENTITY_SERDE;
 import static org.apache.gravitino.Configs.ENTITY_STORE;
 import static org.apache.gravitino.Configs.RELATIONAL_ENTITY_STORE;
 import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME;
@@ -184,7 +183,6 @@ public static void setUp() {
     when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L);
     when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
     when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L);
-    when(config.get(ENTITY_SERDE)).thenReturn("proto");
 
     store = EntityStoreFactory.createEntityStore(config);
     store.initialize(config);

diff --git a/...og-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/...og-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java
@@ -19,17 +19,13 @@
 package org.apache.gravitino.catalog.kafka;
 
 import static org.apache.gravitino.Catalog.Type.MESSAGING;
-import static org.apache.gravitino.Configs.DEFAULT_ENTITY_KV_STORE;
 import static org.apache.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE;
-import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH;
-import static org.apache.gravitino.Configs.ENTITY_KV_STORE;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER;
 import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_STORE;
-import static org.apache.gravitino.Configs.ENTITY_SERDE;
 import static org.apache.gravitino.Configs.ENTITY_STORE;
 import static org.apache.gravitino.Configs.RELATIONAL_ENTITY_STORE;
 import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME;
@@ -126,12 +122,6 @@ public PropertiesMetadata topicPropertiesMetadata() throws UnsupportedOperationE
   @BeforeAll
   public static void setUp() {
     Config config = Mockito.mock(Config.class);
-    Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv");
-    Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE);
-    Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto");
-    Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(STORE_PATH);
-
-    Assertions.assertEquals(STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH));
     Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
     Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L);
 
@@ -151,7 +141,6 @@ public static void setUp() {
     when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L);
     when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L);
     when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L);
-    when(config.get(ENTITY_SERDE)).thenReturn("proto");
 
     // Mock
     MetalakeMetaService metalakeMetaService = MetalakeMetaService.getInstance();

diff --git a/core/build.gradle.kts b/core/build.gradle.kts
@@ -26,7 +26,6 @@ dependencies {
   implementation(project(":api"))
   implementation(project(":common"))
   implementation(project(":catalogs:catalog-common"))
-  implementation(project(":meta"))
   implementation(libs.bundles.log4j)
   implementation(libs.bundles.metrics)
   implementation(libs.bundles.prometheus)
@@ -41,7 +40,6 @@ dependencies {
     exclude("com.google.guava", "guava")
       .because("Brings in Guava for Android, which we don't want (and breaks multimaps).")
   }
-  implementation(libs.rocksdbjni)
 
   annotationProcessor(libs.lombok)
 

diff --git a/core/src/main/java/org/apache/gravitino/Config.java b/core/src/main/java/org/apache/gravitino/Config.java
@@ -49,10 +49,6 @@ public abstract class Config {
   // Constant Array to hold all deprecated configuration keys, when a configuration is deprecated,
   // we should add it here.
   private final DeprecatedConfig[] deprecatedConfigs = {
-    new DeprecatedConfig(
-        "gravitino.entity.store.kv.deleteAfterTimeMs",
-        "0.5.0",
-        "Please use gravitino.entity.store.deleteAfterTimeMs instead."),
     new DeprecatedConfig(
         "gravitino.authenticator", "0.6.0", "Please use gravitino.authenticators instead.")
   };

diff --git a/core/src/main/java/org/apache/gravitino/Configs.java b/core/src/main/java/org/apache/gravitino/Configs.java
@@ -33,13 +33,9 @@ public class Configs {
 
   private Configs() {}
 
-  public static final String KV_STORE_KEY = "kv";
   public static final String RELATIONAL_ENTITY_STORE = "relational";
   public static final String ENTITY_STORE_KEY = "gravitino.entity.store";
 
-  public static final String DEFAULT_ENTITY_KV_STORE = "RocksDBKvBackend";
-  public static final String ENTITY_KV_STORE_KEY = "gravitino.entity.store.kv";
-
   public static final String DEFAULT_ENTITY_RELATIONAL_STORE = "JDBCBackend";
   public static final String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational";
 
@@ -55,18 +51,13 @@ private Configs() {}
   public static final String ENTITY_RELATIONAL_JDBC_BACKEND_STORAGE_PATH_KEY =
       "gravitino.entity.store.relational.storagePath";
 
-  public static final String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY =
-      "gravitino.entity.store.kv.rocksdbPath";
-
-  public static final Long DEFAULT_KV_DELETE_AFTER_TIME = 604800000L; // 7 days
-  public static final String KV_DELETE_AFTER_TIME_KEY =
-      "gravitino.entity.store.kv.deleteAfterTimeMs";
+  public static final Long DEFAULT_DELETE_AFTER_TIME = 604800000L; // 7 days
 
   // Config for data keep time after soft deletion, in milliseconds.
   public static final String STORE_DELETE_AFTER_TIME_KEY =
       "gravitino.entity.store.deleteAfterTimeMs";
   // using the fallback default value
-  public static final Long DEFAULT_STORE_DELETE_AFTER_TIME = DEFAULT_KV_DELETE_AFTER_TIME;
+  public static final Long DEFAULT_STORE_DELETE_AFTER_TIME = DEFAULT_DELETE_AFTER_TIME;
   // The maximum allowed keep time for data deletion, in milliseconds. Equivalent to 30 days.
   public static final Long MAX_DELETE_TIME_ALLOW = 1000 * 60 * 60 * 24 * 30L;
   // The minimum allowed keep time for data deletion, in milliseconds. Equivalent to 10 minutes.
@@ -82,10 +73,6 @@ private Configs() {}
   // The minimum allowed count of versions to be retained
   public static final Long MIN_VERSION_RETENTION_COUNT = 1L;
 
-  // Default path for RocksDB backend is "${GRAVITINO_HOME}/data/rocksdb"
-  public static final String DEFAULT_KV_ROCKSDB_BACKEND_PATH =
-      String.join(File.separator, System.getenv("GRAVITINO_HOME"), "data", "rocksdb");
-
   public static final String DEFAULT_RELATIONAL_JDBC_BACKEND_PATH =
       String.join(File.separator, System.getenv("GRAVITINO_HOME"), "data", "jdbc");
 
@@ -111,13 +98,6 @@ private Configs() {}
           .stringConf()
           .createWithDefault(RELATIONAL_ENTITY_STORE);
 
-  public static final ConfigEntry<String> ENTITY_KV_STORE =
-      new ConfigBuilder(ENTITY_KV_STORE_KEY)
-          .doc("Detailed implementation of Kv storage")
-          .version(ConfigConstants.VERSION_0_1_0)
-          .stringConf()
-          .createWithDefault(DEFAULT_ENTITY_KV_STORE);
-
   public static final ConfigEntry<String> ENTITY_RELATIONAL_STORE =
       new ConfigBuilder(ENTITY_RELATIONAL_STORE_KEY)
           .doc("Detailed implementation of relational storage")
@@ -168,24 +148,6 @@ private Configs() {}
           .stringConf()
           .createWithDefault(DEFAULT_RELATIONAL_JDBC_BACKEND_PATH);
 
-  public static final ConfigEntry<String> ENTITY_KV_ROCKSDB_BACKEND_PATH =
-      new ConfigBuilder(ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY)
-          .doc(
-              "The storage path for RocksDB storage implementation. It supports both absolute and"
-                  + " relative path, if the value is a relative path, the final path is "
-                  + "`${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is "
-                  + "`${GRAVITINO_HOME}/data/rocksdb`")
-          .version(ConfigConstants.VERSION_0_1_0)
-          .stringConf()
-          .createWithDefault(DEFAULT_KV_ROCKSDB_BACKEND_PATH);
-
-  public static final ConfigEntry<String> ENTITY_SERDE =
-      new ConfigBuilder("gravitino.entity.serde")
-          .doc("The entity SerDe to use")
-          .version(ConfigConstants.VERSION_0_1_0)
-          .stringConf()
-          .createWithDefault("proto");
-
   public static final ConfigEntry<Long> CATALOG_CACHE_EVICTION_INTERVAL_MS =
       new ConfigBuilder("gravitino.catalog.cache.evictionIntervalMs")
           .doc("The interval in milliseconds to evict the catalog cache")
@@ -232,24 +194,6 @@ private Configs() {}
           .longConf()
           .createWithDefault(2000L);
 
-  public static final ConfigEntry<Long> KV_DELETE_AFTER_TIME =
-      new ConfigBuilder(KV_DELETE_AFTER_TIME_KEY)
-          .doc(
-              String.format(
-                  "The maximum time in milliseconds that the deleted data and old version data is kept, "
-                      + "max delete time allow is %s ms(30 days), "
-                      + "min delete time allow is %s ms(10 minutes)",
-                  MAX_DELETE_TIME_ALLOW, MIN_DELETE_TIME_ALLOW))
-          .version(ConfigConstants.VERSION_0_5_0)
-          .deprecated()
-          .longConf()
-          .checkValue(
-              v -> v >= MIN_DELETE_TIME_ALLOW && v <= MAX_DELETE_TIME_ALLOW,
-              String.format(
-                  "The value of %s is out of range, which must be between %s and %s",
-                  KV_DELETE_AFTER_TIME_KEY, MIN_DELETE_TIME_ALLOW, MAX_DELETE_TIME_ALLOW))
-          .createWithDefault(DEFAULT_KV_DELETE_AFTER_TIME);
-
   public static final ConfigEntry<Long> STORE_DELETE_AFTER_TIME =
       new ConfigBuilder(STORE_DELETE_AFTER_TIME_KEY)
           .doc(
@@ -259,7 +203,6 @@ private Configs() {}
                       + "min delete time allow is %s ms(10 minutes)",
                   MAX_DELETE_TIME_ALLOW, MIN_DELETE_TIME_ALLOW))
           .version(ConfigConstants.VERSION_0_5_0)
-          .alternatives(Lists.newArrayList(KV_DELETE_AFTER_TIME_KEY))
           .longConf()
           .checkValue(
               v -> v >= MIN_DELETE_TIME_ALLOW && v <= MAX_DELETE_TIME_ALLOW,

diff --git a/core/src/main/java/org/apache/gravitino/EntitySerDe.java b/core/src/main/java/org/apache/gravitino/EntitySerDe.java
-Original file line number
+Diff line change
@@ Expand Up / @@ -384,7 +384,6 @@ @@
        JSR305
        LevelDB JNI
-       RocksDB JNI
        JCraft
        Javolution
        ANTLR4
@@ Expand Down @@