From 46cae48b84b967dcfdcb2926f47d6aa6a6bda784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A7=80=ED=99=8D=EA=B7=BC=5BData=20Analytics=20Solution?= =?UTF-8?q?=5D?= Date: Fri, 22 Nov 2024 11:04:10 +0900 Subject: [PATCH] [#5633] remove the key-value storage backend logic --- LICENSE.bin | 1 - LICENSE.rest | 1 - build.gradle.kts | 88 +- .../kafka/TestKafkaCatalogOperations.java | 8 - core/build.gradle.kts | 2 - .../java/org/apache/gravitino/Config.java | 4 - .../java/org/apache/gravitino/Configs.java | 61 +- .../org/apache/gravitino/EntitySerDe.java | 68 -- .../apache/gravitino/EntitySerDeFactory.java | 71 -- .../org/apache/gravitino/EntityStore.java | 8 - .../apache/gravitino/EntityStoreFactory.java | 13 +- .../gravitino/authorization/OwnerManager.java | 9 +- .../gravitino/proto/AuditInfoSerDe.java | 71 -- .../gravitino/proto/BaseMetalakeSerDe.java | 91 -- .../gravitino/proto/CatalogEntitySerDe.java | 87 -- .../gravitino/proto/FilesetEntitySerDe.java | 69 -- .../gravitino/proto/GroupEntitySerDe.java | 69 -- .../gravitino/proto/ProtoEntitySerDe.java | 177 ---- .../apache/gravitino/proto/ProtoSerDe.java | 48 - .../gravitino/proto/ProtoSerDeException.java | 44 - .../apache/gravitino/proto/ProtoUtils.java | 51 -- .../gravitino/proto/RoleEntitySerDe.java | 115 --- .../gravitino/proto/SchemaEntitySerDe.java | 63 -- .../gravitino/proto/TableEntitySerDe.java | 43 - .../gravitino/proto/TopicEntitySerDe.java | 64 -- .../gravitino/proto/UserEntitySerDe.java | 69 -- .../apache/gravitino/storage/BiPredicate.java | 31 - .../gravitino/storage/EntityKeyEncoder.java | 62 -- .../gravitino/storage/FunctionUtils.java | 80 -- .../gravitino/storage/NameMappingService.java | 91 -- .../storage/StorageLayoutException.java | 50 -- .../storage/StorageLayoutVersion.java | 44 - .../storage/TransactionIdGenerator.java | 37 - .../storage/kv/BinaryEntityEncoderUtil.java | 204 ----- .../storage/kv/BinaryEntityKeyEncoder.java | 272 ------ .../gravitino/storage/kv/KvBackend.java | 86 -- .../gravitino/storage/kv/KvEntityStore.java | 360 -------- .../storage/kv/KvGarbageCollector.java | 403 --------- .../storage/kv/KvNameMappingService.java | 172 ---- .../apache/gravitino/storage/kv/KvRange.java | 74 -- .../storage/kv/RocksDBKvBackend.java | 246 ----- .../kv/TransactionIdGeneratorImpl.java | 164 ---- .../storage/kv/TransactionalKvBackend.java | 52 -- .../kv/TransactionalKvBackendImpl.java | 400 --------- .../gravitino/storage/kv/ValueStatusEnum.java | 53 -- .../relational/RelationalEntityStore.java | 6 - .../org/apache/gravitino/tag/TagManager.java | 9 - ...estAccessControlManagerForPermissions.java | 1 - .../gravitino/catalog/TestCatalogManager.java | 1 - .../TestCatalogNormalizeDispatcher.java | 1 - .../catalog/TestOperationDispatcher.java | 1 - .../metalake/TestMetalakeManager.java | 1 - .../TestMetalakeNormalizeDispatcher.java | 1 - .../gravitino/proto/TestEntityProtoSerDe.java | 432 --------- .../gravitino/storage/TestEntityStorage.java | 98 +- .../storage/kv/TestEntityKeyEncoding.java | 420 --------- .../storage/kv/TestKvEntityStorage.java | 304 ------- .../storage/kv/TestKvGarbageCollector.java | 522 ----------- .../storage/kv/TestKvNameMappingService.java | 186 ---- .../storage/kv/TestRocksDBKvBackend.java | 461 ---------- .../storage/kv/TestStorageVersion.java | 87 -- .../kv/TestTransactionIdGenerator.java | 146 --- .../kv/TestTransactionalKvBackend.java | 848 ------------------ .../storage/memory/TestMemoryEntityStore.java | 5 - docs/glossary.md | 4 - docs/gravitino-server-config.md | 2 - gradle/libs.versions.toml | 2 - .../integration/test/MiniGravitino.java | 6 - meta/build.gradle.kts | 40 - meta/src/main/proto/gravitino_meta.proto | 148 --- ...y-key-encoding-for-kv-store-deprecated.md} | 8 +- rfc/rfc-3/Transaction-implementation-on-kv.md | 140 --- .../gravitino/server/TestGravitinoServer.java | 19 - settings.gradle.kts | 2 +- 74 files changed, 56 insertions(+), 8121 deletions(-) delete mode 100644 core/src/main/java/org/apache/gravitino/EntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/EntitySerDeFactory.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/AuditInfoSerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/CatalogEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/FilesetEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/ProtoSerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/ProtoSerDeException.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/ProtoUtils.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/SchemaEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/TableEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/TopicEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/BiPredicate.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/EntityKeyEncoder.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/FunctionUtils.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/NameMappingService.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/StorageLayoutException.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/StorageLayoutVersion.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/TransactionIdGenerator.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityEncoderUtil.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/KvBackend.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/KvEntityStore.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/KvGarbageCollector.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/KvNameMappingService.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/KvRange.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/RocksDBKvBackend.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/TransactionIdGeneratorImpl.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackend.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackendImpl.java delete mode 100644 core/src/main/java/org/apache/gravitino/storage/kv/ValueStatusEnum.java delete mode 100644 core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestEntityKeyEncoding.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestKvEntityStorage.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestKvGarbageCollector.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestKvNameMappingService.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestRocksDBKvBackend.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestStorageVersion.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionIdGenerator.java delete mode 100644 core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionalKvBackend.java delete mode 100644 meta/build.gradle.kts delete mode 100644 meta/src/main/proto/gravitino_meta.proto rename rfc/rfc-2/{Entity-key-encoding-for-kv-store.md => Entity-key-encoding-for-kv-store-deprecated.md} (96%) delete mode 100644 rfc/rfc-3/Transaction-implementation-on-kv.md diff --git a/LICENSE.bin b/LICENSE.bin index 9ab5edbd6fc..34723024e78 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -384,7 +384,6 @@ JSR305 LevelDB JNI - RocksDB JNI JCraft Javolution ANTLR4 diff --git a/LICENSE.rest b/LICENSE.rest index 18c7b9c6b77..0d770dc52ce 100644 --- a/LICENSE.rest +++ b/LICENSE.rest @@ -314,7 +314,6 @@ JSR305 LevelDB JNI Protobuf - RocksDB JNI Stax2 API XMLenc diff --git a/build.gradle.kts b/build.gradle.kts index cc29ff4affc..88cdfc6ca32 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -303,51 +303,49 @@ subprojects { } } - if (project.name != "meta") { - apply(plugin = "net.ltgt.errorprone") - dependencies { - errorprone("com.google.errorprone:error_prone_core:2.10.0") - } - - tasks.withType().configureEach { - options.errorprone.isEnabled.set(true) - options.errorprone.disableWarningsInGeneratedCode.set(true) - options.errorprone.disable( - "AlmostJavadoc", - "CanonicalDuration", - "CheckReturnValue", - "ComparableType", - "ConstantOverflow", - "DoubleBraceInitialization", - "EqualsUnsafeCast", - "EmptyBlockTag", - "FutureReturnValueIgnored", - "InconsistentCapitalization", - "InconsistentHashCode", - "JavaTimeDefaultTimeZone", - "JdkObsolete", - "LockNotBeforeTry", - "MissingSummary", - "MissingOverride", - "MutableConstantField", - "NonOverridingEquals", - "ObjectEqualsForPrimitives", - "OperatorPrecedence", - "ReturnValueIgnored", - "SameNameButDifferent", - "StaticAssignmentInConstructor", - "StringSplitter", - "ThreadPriorityCheck", - "ThrowIfUncheckedKnownChecked", - "TypeParameterUnusedInFormals", - "UnicodeEscape", - "UnnecessaryParentheses", - "UnsafeReflectiveConstructionCast", - "UnusedMethod", - "VariableNameSameAsType", - "WaitNotInLoop" - ) - } + apply(plugin = "net.ltgt.errorprone") + dependencies { + errorprone("com.google.errorprone:error_prone_core:2.10.0") + } + + tasks.withType().configureEach { + options.errorprone.isEnabled.set(true) + options.errorprone.disableWarningsInGeneratedCode.set(true) + options.errorprone.disable( + "AlmostJavadoc", + "CanonicalDuration", + "CheckReturnValue", + "ComparableType", + "ConstantOverflow", + "DoubleBraceInitialization", + "EqualsUnsafeCast", + "EmptyBlockTag", + "FutureReturnValueIgnored", + "InconsistentCapitalization", + "InconsistentHashCode", + "JavaTimeDefaultTimeZone", + "JdkObsolete", + "LockNotBeforeTry", + "MissingSummary", + "MissingOverride", + "MutableConstantField", + "NonOverridingEquals", + "ObjectEqualsForPrimitives", + "OperatorPrecedence", + "ReturnValueIgnored", + "SameNameButDifferent", + "StaticAssignmentInConstructor", + "StringSplitter", + "ThreadPriorityCheck", + "ThrowIfUncheckedKnownChecked", + "TypeParameterUnusedInFormals", + "UnicodeEscape", + "UnnecessaryParentheses", + "UnsafeReflectiveConstructionCast", + "UnusedMethod", + "VariableNameSameAsType", + "WaitNotInLoop" + ) } tasks.withType { diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java index 58cd550425f..a27c117a134 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java @@ -19,10 +19,7 @@ package org.apache.gravitino.catalog.kafka; import static org.apache.gravitino.Catalog.Type.MESSAGING; -import static org.apache.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; import static org.apache.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.ENTITY_KV_STORE; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH; @@ -126,12 +123,7 @@ public PropertiesMetadata topicPropertiesMetadata() throws UnsupportedOperationE @BeforeAll public static void setUp() { Config config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); - Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(STORE_PATH); - - Assertions.assertEquals(STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); diff --git a/core/build.gradle.kts b/core/build.gradle.kts index b09c0e35889..a0468168b69 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -26,7 +26,6 @@ dependencies { implementation(project(":api")) implementation(project(":common")) implementation(project(":catalogs:catalog-common")) - implementation(project(":meta")) implementation(libs.bundles.log4j) implementation(libs.bundles.metrics) implementation(libs.bundles.prometheus) @@ -41,7 +40,6 @@ dependencies { exclude("com.google.guava", "guava") .because("Brings in Guava for Android, which we don't want (and breaks multimaps).") } - implementation(libs.rocksdbjni) annotationProcessor(libs.lombok) diff --git a/core/src/main/java/org/apache/gravitino/Config.java b/core/src/main/java/org/apache/gravitino/Config.java index de61a44d165..642cf8bfdc9 100644 --- a/core/src/main/java/org/apache/gravitino/Config.java +++ b/core/src/main/java/org/apache/gravitino/Config.java @@ -49,10 +49,6 @@ public abstract class Config { // Constant Array to hold all deprecated configuration keys, when a configuration is deprecated, // we should add it here. private final DeprecatedConfig[] deprecatedConfigs = { - new DeprecatedConfig( - "gravitino.entity.store.kv.deleteAfterTimeMs", - "0.5.0", - "Please use gravitino.entity.store.deleteAfterTimeMs instead."), new DeprecatedConfig( "gravitino.authenticator", "0.6.0", "Please use gravitino.authenticators instead.") }; diff --git a/core/src/main/java/org/apache/gravitino/Configs.java b/core/src/main/java/org/apache/gravitino/Configs.java index 3ff4c7c4d86..1a94e47e392 100644 --- a/core/src/main/java/org/apache/gravitino/Configs.java +++ b/core/src/main/java/org/apache/gravitino/Configs.java @@ -33,13 +33,9 @@ public class Configs { private Configs() {} - public static final String KV_STORE_KEY = "kv"; public static final String RELATIONAL_ENTITY_STORE = "relational"; public static final String ENTITY_STORE_KEY = "gravitino.entity.store"; - public static final String DEFAULT_ENTITY_KV_STORE = "RocksDBKvBackend"; - public static final String ENTITY_KV_STORE_KEY = "gravitino.entity.store.kv"; - public static final String DEFAULT_ENTITY_RELATIONAL_STORE = "JDBCBackend"; public static final String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; @@ -55,18 +51,13 @@ private Configs() {} public static final String ENTITY_RELATIONAL_JDBC_BACKEND_STORAGE_PATH_KEY = "gravitino.entity.store.relational.storagePath"; - public static final String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = - "gravitino.entity.store.kv.rocksdbPath"; - - public static final Long DEFAULT_KV_DELETE_AFTER_TIME = 604800000L; // 7 days - public static final String KV_DELETE_AFTER_TIME_KEY = - "gravitino.entity.store.kv.deleteAfterTimeMs"; + public static final Long DEFAULT_DELETE_AFTER_TIME = 604800000L; // 7 days // Config for data keep time after soft deletion, in milliseconds. public static final String STORE_DELETE_AFTER_TIME_KEY = "gravitino.entity.store.deleteAfterTimeMs"; // using the fallback default value - public static final Long DEFAULT_STORE_DELETE_AFTER_TIME = DEFAULT_KV_DELETE_AFTER_TIME; + public static final Long DEFAULT_STORE_DELETE_AFTER_TIME = DEFAULT_DELETE_AFTER_TIME; // The maximum allowed keep time for data deletion, in milliseconds. Equivalent to 30 days. public static final Long MAX_DELETE_TIME_ALLOW = 1000 * 60 * 60 * 24 * 30L; // The minimum allowed keep time for data deletion, in milliseconds. Equivalent to 10 minutes. @@ -82,10 +73,6 @@ private Configs() {} // The minimum allowed count of versions to be retained public static final Long MIN_VERSION_RETENTION_COUNT = 1L; - // Default path for RocksDB backend is "${GRAVITINO_HOME}/data/rocksdb" - public static final String DEFAULT_KV_ROCKSDB_BACKEND_PATH = - String.join(File.separator, System.getenv("GRAVITINO_HOME"), "data", "rocksdb"); - public static final String DEFAULT_RELATIONAL_JDBC_BACKEND_PATH = String.join(File.separator, System.getenv("GRAVITINO_HOME"), "data", "jdbc"); @@ -111,13 +98,6 @@ private Configs() {} .stringConf() .createWithDefault(RELATIONAL_ENTITY_STORE); - public static final ConfigEntry ENTITY_KV_STORE = - new ConfigBuilder(ENTITY_KV_STORE_KEY) - .doc("Detailed implementation of Kv storage") - .version(ConfigConstants.VERSION_0_1_0) - .stringConf() - .createWithDefault(DEFAULT_ENTITY_KV_STORE); - public static final ConfigEntry ENTITY_RELATIONAL_STORE = new ConfigBuilder(ENTITY_RELATIONAL_STORE_KEY) .doc("Detailed implementation of relational storage") @@ -168,24 +148,6 @@ private Configs() {} .stringConf() .createWithDefault(DEFAULT_RELATIONAL_JDBC_BACKEND_PATH); - public static final ConfigEntry ENTITY_KV_ROCKSDB_BACKEND_PATH = - new ConfigBuilder(ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY) - .doc( - "The storage path for RocksDB storage implementation. It supports both absolute and" - + " relative path, if the value is a relative path, the final path is " - + "`${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is " - + "`${GRAVITINO_HOME}/data/rocksdb`") - .version(ConfigConstants.VERSION_0_1_0) - .stringConf() - .createWithDefault(DEFAULT_KV_ROCKSDB_BACKEND_PATH); - - public static final ConfigEntry ENTITY_SERDE = - new ConfigBuilder("gravitino.entity.serde") - .doc("The entity SerDe to use") - .version(ConfigConstants.VERSION_0_1_0) - .stringConf() - .createWithDefault("proto"); - public static final ConfigEntry CATALOG_CACHE_EVICTION_INTERVAL_MS = new ConfigBuilder("gravitino.catalog.cache.evictionIntervalMs") .doc("The interval in milliseconds to evict the catalog cache") @@ -232,24 +194,6 @@ private Configs() {} .longConf() .createWithDefault(2000L); - public static final ConfigEntry KV_DELETE_AFTER_TIME = - new ConfigBuilder(KV_DELETE_AFTER_TIME_KEY) - .doc( - String.format( - "The maximum time in milliseconds that the deleted data and old version data is kept, " - + "max delete time allow is %s ms(30 days), " - + "min delete time allow is %s ms(10 minutes)", - MAX_DELETE_TIME_ALLOW, MIN_DELETE_TIME_ALLOW)) - .version(ConfigConstants.VERSION_0_5_0) - .deprecated() - .longConf() - .checkValue( - v -> v >= MIN_DELETE_TIME_ALLOW && v <= MAX_DELETE_TIME_ALLOW, - String.format( - "The value of %s is out of range, which must be between %s and %s", - KV_DELETE_AFTER_TIME_KEY, MIN_DELETE_TIME_ALLOW, MAX_DELETE_TIME_ALLOW)) - .createWithDefault(DEFAULT_KV_DELETE_AFTER_TIME); - public static final ConfigEntry STORE_DELETE_AFTER_TIME = new ConfigBuilder(STORE_DELETE_AFTER_TIME_KEY) .doc( @@ -259,7 +203,6 @@ private Configs() {} + "min delete time allow is %s ms(10 minutes)", MAX_DELETE_TIME_ALLOW, MIN_DELETE_TIME_ALLOW)) .version(ConfigConstants.VERSION_0_5_0) - .alternatives(Lists.newArrayList(KV_DELETE_AFTER_TIME_KEY)) .longConf() .checkValue( v -> v >= MIN_DELETE_TIME_ALLOW && v <= MAX_DELETE_TIME_ALLOW, diff --git a/core/src/main/java/org/apache/gravitino/EntitySerDe.java b/core/src/main/java/org/apache/gravitino/EntitySerDe.java deleted file mode 100644 index 722d946aa10..00000000000 --- a/core/src/main/java/org/apache/gravitino/EntitySerDe.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino; - -import java.io.IOException; -import java.util.Optional; - -public interface EntitySerDe { - - /** - * Serializes the entity to a byte array. - * - * @param t the entity to serialize - * @return the serialized byte array of the entity - * @param The type of entity - * @throws IOException if the serialization fails - */ - byte[] serialize(T t) throws IOException; - - /** - * Deserializes the entity from a byte array. - * - * @param bytes the byte array to deserialize - * @param clazz the class of the entity - * @param namespace the namespace to use when deserializing the entity - * @return the deserialized entity - * @param The type of entity - * @throws IOException if the deserialization fails - */ - default T deserialize(byte[] bytes, Class clazz, Namespace namespace) - throws IOException { - ClassLoader loader = - Optional.ofNullable(Thread.currentThread().getContextClassLoader()) - .orElse(getClass().getClassLoader()); - return deserialize(bytes, clazz, loader, namespace); - } - - /** - * Deserializes the entity from a byte array. - * - * @param bytes the byte array to deserialize - * @param clazz the class of the entity - * @param classLoader the class loader to use - * @param namespace the namespace to use when deserializing the entity - * @return the deserialized entity - * @param The type of entity - * @throws IOException if the deserialization fails - */ - T deserialize( - byte[] bytes, Class clazz, ClassLoader classLoader, Namespace namespace) - throws IOException; -} diff --git a/core/src/main/java/org/apache/gravitino/EntitySerDeFactory.java b/core/src/main/java/org/apache/gravitino/EntitySerDeFactory.java deleted file mode 100644 index dee0cdc0fee..00000000000 --- a/core/src/main/java/org/apache/gravitino/EntitySerDeFactory.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino; - -import com.google.common.collect.ImmutableMap; -import java.util.Map; -import org.apache.gravitino.proto.ProtoEntitySerDe; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is responsible for creating instances of EntitySerDe implementations. EntitySerDe - * (Entity Serialization/Deserialization) implementations are used to serialize and deserialize - * entities within the Apache Gravitino framework. - */ -public class EntitySerDeFactory { - - private static final Logger LOG = LoggerFactory.getLogger(EntitySerDeFactory.class); - - // Register EntitySerDe's short name to its full qualified class name in the map. So that user - // don't need to specify the full qualified class name when creating an EntitySerDe. - private static final Map ENTITY_SERDES = - ImmutableMap.of("proto", ProtoEntitySerDe.class.getCanonicalName()); - - private EntitySerDeFactory() {} - - /** - * Creates an instance of EntitySerDe. - * - * @param config The configuration object containing settings for EntitySerDe. - * @return An instance of EntitySerDe. - */ - public static EntitySerDe createEntitySerDe(Config config) { - String name = config.get(Configs.ENTITY_SERDE); - return createEntitySerDe(name); - } - - /** - * Creates an instance of EntitySerDe. - * - * @param name The short name identifying the EntitySerDe implementation. - * @return An instance of EntitySerDe. - * @throws RuntimeException If the EntitySerDe creation fails. - */ - public static EntitySerDe createEntitySerDe(String name) { - String className = ENTITY_SERDES.getOrDefault(name, name); - - try { - return (EntitySerDe) Class.forName(className).getDeclaredConstructor().newInstance(); - } catch (Exception e) { - LOG.error("Failed to create EntitySerDe by name {}.", name, e); - throw new RuntimeException("Failed to create EntitySerDe: " + name, e); - } - } -} diff --git a/core/src/main/java/org/apache/gravitino/EntityStore.java b/core/src/main/java/org/apache/gravitino/EntityStore.java index d26dbf13b5a..2993cefe983 100644 --- a/core/src/main/java/org/apache/gravitino/EntityStore.java +++ b/core/src/main/java/org/apache/gravitino/EntityStore.java @@ -40,14 +40,6 @@ public interface EntityStore extends Closeable { */ void initialize(Config config) throws RuntimeException; - /** - * Set the {@link EntitySerDe} for the entity store. {@link EntitySerDe} will be used to serialize - * and deserialize the entities to the target format, and vice versa. - * - * @param entitySerDe the entity serde to set - */ - void setSerDe(EntitySerDe entitySerDe); - /** * List all the entities with the specified {@link org.apache.gravitino.Namespace}, and * deserialize them into the specified {@link Entity} object. diff --git a/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java b/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java index 372e5faf4bb..7bc0ddd1853 100644 --- a/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/org/apache/gravitino/EntityStoreFactory.java @@ -18,10 +18,7 @@ */ package org.apache.gravitino; -import static org.apache.gravitino.Configs.KV_STORE_KEY; - import com.google.common.collect.ImmutableMap; -import org.apache.gravitino.storage.kv.KvEntityStore; import org.apache.gravitino.storage.relational.RelationalEntityStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,10 +35,7 @@ public class EntityStoreFactory { // doesn't need to specify the full qualified class name when creating an EntityStore. public static final ImmutableMap ENTITY_STORES = ImmutableMap.of( - KV_STORE_KEY, - KvEntityStore.class.getCanonicalName(), - Configs.RELATIONAL_ENTITY_STORE, - RelationalEntityStore.class.getCanonicalName()); + Configs.RELATIONAL_ENTITY_STORE, RelationalEntityStore.class.getCanonicalName()); // Private constructor to prevent instantiation of this factory class. private EntityStoreFactory() {} @@ -56,11 +50,6 @@ public static EntityStore createEntityStore(Config config) { String name = config.get(Configs.ENTITY_STORE); String className = ENTITY_STORES.getOrDefault(name, name); - if (KV_STORE_KEY.equals(name)) { - throw new UnsupportedOperationException( - "KvEntityStore is not supported since version 0.6.0. Please use RelationalEntityStore instead."); - } - try { return (EntityStore) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { diff --git a/core/src/main/java/org/apache/gravitino/authorization/OwnerManager.java b/core/src/main/java/org/apache/gravitino/authorization/OwnerManager.java index 04285954e97..fa4e6786547 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/OwnerManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/OwnerManager.java @@ -33,7 +33,6 @@ import org.apache.gravitino.lock.TreeLockUtils; import org.apache.gravitino.meta.GroupEntity; import org.apache.gravitino.meta.UserEntity; -import org.apache.gravitino.storage.kv.KvEntityStore; import org.apache.gravitino.utils.MetadataObjectUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,13 +47,7 @@ public class OwnerManager { private final EntityStore store; public OwnerManager(EntityStore store) { - if (store instanceof KvEntityStore) { - String errorMsg = - "OwnerManager cannot run with kv entity store, please configure the entity " - + "store to use relational entity store and restart the Gravitino server"; - LOG.error(errorMsg); - throw new RuntimeException(errorMsg); - } else if (store instanceof SupportsRelationOperations) { + if (store instanceof SupportsRelationOperations) { this.store = store; } else { String errorMsg = diff --git a/core/src/main/java/org/apache/gravitino/proto/AuditInfoSerDe.java b/core/src/main/java/org/apache/gravitino/proto/AuditInfoSerDe.java deleted file mode 100644 index c8a7f1dbf6f..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/AuditInfoSerDe.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.proto; - -import java.util.Optional; -import org.apache.gravitino.Namespace; - -/** A class for serializing and deserializing AuditInfo objects. */ -class AuditInfoSerDe implements ProtoSerDe { - - /** - * Serializes an {@link org.apache.gravitino.meta.AuditInfo} object to a {@link AuditInfo} object. - * - * @param auditInfo The AuditInfo object to be serialized. - * @return The serialized AuditInfo object. - */ - @Override - public AuditInfo serialize(org.apache.gravitino.meta.AuditInfo auditInfo) { - AuditInfo.Builder builder = AuditInfo.newBuilder(); - - Optional.ofNullable(auditInfo.creator()).ifPresent(builder::setCreator); - Optional.ofNullable(auditInfo.createTime()) - .map(ProtoUtils::fromInstant) - .ifPresent(builder::setCreateTime); - Optional.ofNullable(auditInfo.lastModifier()).ifPresent(builder::setLastModifier); - Optional.ofNullable(auditInfo.lastModifiedTime()) - .map(ProtoUtils::fromInstant) - .ifPresent(builder::setLastModifiedTime); - - return builder.build(); - } - - /** - * Deserializes a {@link AuditInfo} object to an {@link org.apache.gravitino.meta.AuditInfo} - * object. - * - * @param p The serialized AuditInfo object. - * @return The deserialized AuditInfo object. - */ - @Override - public org.apache.gravitino.meta.AuditInfo deserialize(AuditInfo p, Namespace namespace) { - org.apache.gravitino.meta.AuditInfo.Builder builder = - org.apache.gravitino.meta.AuditInfo.builder(); - - if (p.hasCreator()) builder.withCreator(p.getCreator()); - if (p.hasCreateTime()) builder.withCreateTime(ProtoUtils.toInstant(p.getCreateTime())); - if (p.hasLastModifier()) builder.withLastModifier(p.getLastModifier()); - if (p.hasLastModifiedTime()) { - builder.withLastModifiedTime(ProtoUtils.toInstant(p.getLastModifiedTime())); - } - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java b/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java deleted file mode 100644 index e18b280e691..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.proto; - -import org.apache.gravitino.Namespace; - -/** A class for serializing and deserializing BaseMetalake objects. */ -class BaseMetalakeSerDe implements ProtoSerDe { - - /** - * Serializes a {@link org.apache.gravitino.meta.BaseMetalake} object to a {@link Metalake} - * object. - * - * @param baseMetalake The BaseMetalake object to be serialized. - * @return The serialized Metalake object. - */ - @Override - public Metalake serialize(org.apache.gravitino.meta.BaseMetalake baseMetalake) { - Metalake.Builder builder = - Metalake.newBuilder() - .setId(baseMetalake.id()) - .setName(baseMetalake.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(baseMetalake.auditInfo())); - - if (baseMetalake.comment() != null) { - builder.setComment(baseMetalake.comment()); - } - - if (baseMetalake.properties() != null && !baseMetalake.properties().isEmpty()) { - builder.putAllProperties(baseMetalake.properties()); - } - - SchemaVersion version = - SchemaVersion.newBuilder() - .setMajorNumber(baseMetalake.getVersion().majorVersion) - .setMinorNumber(baseMetalake.getVersion().minorVersion) - .build(); - builder.setVersion(version); - - return builder.build(); - } - - /** - * Deserializes a {@link Metalake} object to a {@link org.apache.gravitino.meta.BaseMetalake} - * object. - * - * @param p The serialized Metalake object. - * @return The deserialized BaseMetalake object. - */ - @Override - public org.apache.gravitino.meta.BaseMetalake deserialize(Metalake p, Namespace namespace) { - org.apache.gravitino.meta.BaseMetalake.Builder builder = - org.apache.gravitino.meta.BaseMetalake.builder(); - builder - .withId(p.getId()) - .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); - - if (p.hasComment()) { - builder.withComment(p.getComment()); - } - - if (p.getPropertiesCount() > 0) { - builder.withProperties(p.getPropertiesMap()); - } - - org.apache.gravitino.meta.SchemaVersion version = - org.apache.gravitino.meta.SchemaVersion.forValues( - p.getVersion().getMajorNumber(), p.getVersion().getMinorNumber()); - builder.withVersion(version); - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/CatalogEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/CatalogEntitySerDe.java deleted file mode 100644 index e318f027d4a..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/CatalogEntitySerDe.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.proto; - -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.AuditInfo; -import org.apache.gravitino.meta.CatalogEntity; - -/** A class for serializing and deserializing CatalogEntity objects using Protocol Buffers. */ -public class CatalogEntitySerDe implements ProtoSerDe { - - /** - * Serializes a {@link CatalogEntity} object to a {@link Catalog} object. - * - * @param catalogEntity The CatalogEntity object to be serialized. - * @return The serialized Catalog object. - */ - @Override - public Catalog serialize(CatalogEntity catalogEntity) { - Catalog.Builder builder = - Catalog.newBuilder() - .setId(catalogEntity.id()) - .setName(catalogEntity.name()) - .setProvider(catalogEntity.getProvider()) - .setAuditInfo(new AuditInfoSerDe().serialize((AuditInfo) catalogEntity.auditInfo())); - - if (catalogEntity.getComment() != null) { - builder.setComment(catalogEntity.getComment()); - } - - if (catalogEntity.getProperties() != null && !catalogEntity.getProperties().isEmpty()) { - builder.putAllProperties(catalogEntity.getProperties()); - } - - org.apache.gravitino.proto.Catalog.Type type = - org.apache.gravitino.proto.Catalog.Type.valueOf(catalogEntity.getType().name()); - builder.setType(type); - - // Note we have ignored the namespace field here - return builder.build(); - } - - /** - * Deserializes a {@link Catalog} object to a {@link CatalogEntity} object. - * - * @param p The serialized Catalog object. - * @return The deserialized CatalogEntity object. - */ - @Override - public CatalogEntity deserialize(Catalog p, Namespace namespace) { - CatalogEntity.Builder builder = CatalogEntity.builder(); - builder - .withId(p.getId()) - .withName(p.getName()) - .withNamespace(namespace) - .withProvider(p.getProvider()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); - - if (p.hasComment()) { - builder.withComment(p.getComment()); - } - - if (p.getPropertiesCount() > 0) { - builder.withProperties(p.getPropertiesMap()); - } - - builder.withType(org.apache.gravitino.Catalog.Type.valueOf(p.getType().name())); - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/FilesetEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/FilesetEntitySerDe.java deleted file mode 100644 index 388156da9a3..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/FilesetEntitySerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.FilesetEntity; - -public class FilesetEntitySerDe implements ProtoSerDe { - @Override - public Fileset serialize(FilesetEntity filesetEntity) { - Fileset.Builder builder = - Fileset.newBuilder() - .setId(filesetEntity.id()) - .setName(filesetEntity.name()) - .setStorageLocation(filesetEntity.storageLocation()) - .setAuditInfo(new AuditInfoSerDe().serialize(filesetEntity.auditInfo())); - - if (filesetEntity.comment() != null) { - builder.setComment(filesetEntity.comment()); - } - - if (filesetEntity.properties() != null && !filesetEntity.properties().isEmpty()) { - builder.putAllProperties(filesetEntity.properties()); - } - - Fileset.Type type = Fileset.Type.valueOf(filesetEntity.filesetType().name()); - builder.setType(type); - - return builder.build(); - } - - @Override - public FilesetEntity deserialize(Fileset p, Namespace namespace) { - FilesetEntity.Builder builder = - FilesetEntity.builder() - .withId(p.getId()) - .withName(p.getName()) - .withNamespace(namespace) - .withStorageLocation(p.getStorageLocation()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)) - .withFilesetType(org.apache.gravitino.file.Fileset.Type.valueOf(p.getType().name())); - - if (p.hasComment()) { - builder.withComment(p.getComment()); - } - - if (p.getPropertiesCount() > 0) { - builder.withProperties(p.getPropertiesMap()); - } - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java deleted file mode 100644 index 46eb9c90841..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import java.util.Collection; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.GroupEntity; - -public class GroupEntitySerDe implements ProtoSerDe { - - @Override - public Group serialize(GroupEntity groupEntity) { - Group.Builder builder = - Group.newBuilder() - .setId(groupEntity.id()) - .setName(groupEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(groupEntity.auditInfo())); - - if (isCollectionNotEmpty(groupEntity.roles())) { - builder.addAllRoleNames(groupEntity.roles()); - } - - if (isCollectionNotEmpty(groupEntity.roleIds())) { - builder.addAllRoleIds(groupEntity.roleIds()); - } - - return builder.build(); - } - - @Override - public GroupEntity deserialize(Group group, Namespace namespace) { - GroupEntity.Builder builder = - GroupEntity.builder() - .withId(group.getId()) - .withName(group.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo(), namespace)); - - if (group.getRoleNamesCount() > 0) { - builder.withRoleNames(group.getRoleNamesList()); - } - - if (group.getRoleIdsCount() > 0) { - builder.withRoleIds(group.getRoleIdsList()); - } - - return builder.build(); - } - - private boolean isCollectionNotEmpty(Collection collection) { - return collection != null && !collection.isEmpty(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java deleted file mode 100644 index 3dc5cb1c297..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.google.protobuf.Any; -import com.google.protobuf.Message; -import java.io.IOException; -import java.util.Map; -import org.apache.gravitino.Entity; -import org.apache.gravitino.EntitySerDe; -import org.apache.gravitino.Namespace; - -public class ProtoEntitySerDe implements EntitySerDe { - - // The implementation of different entities should also register its class to this map, - // otherwise ProtoEntitySerDe will not be able to deserialize the entity. - private static final Map ENTITY_TO_SERDE = - ImmutableMap.builder() - .put("org.apache.gravitino.meta.AuditInfo", "org.apache.gravitino.proto.AuditInfoSerDe") - .put( - "org.apache.gravitino.meta.BaseMetalake", - "org.apache.gravitino.proto.BaseMetalakeSerDe") - .put( - "org.apache.gravitino.meta.CatalogEntity", - "org.apache.gravitino.proto.CatalogEntitySerDe") - .put( - "org.apache.gravitino.meta.SchemaEntity", - "org.apache.gravitino.proto.SchemaEntitySerDe") - .put( - "org.apache.gravitino.meta.TableEntity", - "org.apache.gravitino.proto.TableEntitySerDe") - .put( - "org.apache.gravitino.meta.FilesetEntity", - "org.apache.gravitino.proto.FilesetEntitySerDe") - .put( - "org.apache.gravitino.meta.TopicEntity", - "org.apache.gravitino.proto.TopicEntitySerDe") - .put("org.apache.gravitino.meta.UserEntity", "org.apache.gravitino.proto.UserEntitySerDe") - .put( - "org.apache.gravitino.meta.GroupEntity", - "org.apache.gravitino.proto.GroupEntitySerDe") - .put("org.apache.gravitino.meta.RoleEntity", "org.apache.gravitino.proto.RoleEntitySerDe") - .build(); - - private static final Map ENTITY_TO_PROTO = - ImmutableMap.of( - "org.apache.gravitino.meta.AuditInfo", - "org.apache.gravitino.proto.AuditInfo", - "org.apache.gravitino.meta.BaseMetalake", - "org.apache.gravitino.proto.Metalake", - "org.apache.gravitino.meta.CatalogEntity", - "org.apache.gravitino.proto.Catalog", - "org.apache.gravitino.meta.SchemaEntity", - "org.apache.gravitino.proto.Schema", - "org.apache.gravitino.meta.TableEntity", - "org.apache.gravitino.proto.Table", - "org.apache.gravitino.meta.FilesetEntity", - "org.apache.gravitino.proto.Fileset", - "org.apache.gravitino.meta.TopicEntity", - "org.apache.gravitino.proto.Topic", - "org.apache.gravitino.meta.UserEntity", - "org.apache.gravitino.proto.User", - "org.apache.gravitino.meta.GroupEntity", - "org.apache.gravitino.proto.Group", - "org.apache.gravitino.meta.RoleEntity", - "org.apache.gravitino.proto.Role"); - - private final Map, ProtoSerDe> - entityToSerDe; - - private final Map, Class> entityToProto; - - public ProtoEntitySerDe() { - this.entityToSerDe = Maps.newConcurrentMap(); - this.entityToProto = Maps.newConcurrentMap(); - } - - @Override - public byte[] serialize(T t) throws IOException { - Any any = Any.pack(toProto(t, Thread.currentThread().getContextClassLoader())); - return any.toByteArray(); - } - - @Override - public T deserialize( - byte[] bytes, Class clazz, ClassLoader classLoader, Namespace namespace) - throws IOException { - Any any = Any.parseFrom(bytes); - Class protoClass = getProtoClass(clazz, classLoader); - - if (!any.is(protoClass)) { - throw new IOException("Invalid proto for entity " + clazz.getName()); - } - - Message anyMessage = any.unpack(protoClass); - return fromProto(anyMessage, clazz, classLoader, namespace); - } - - private ProtoSerDe getProtoSerde( - Class entityClass, ClassLoader classLoader) throws IOException { - if (!ENTITY_TO_SERDE.containsKey(entityClass.getCanonicalName()) - || ENTITY_TO_SERDE.get(entityClass.getCanonicalName()) == null) { - throw new IOException("No serde found for entity " + entityClass.getCanonicalName()); - } - return (ProtoSerDe) - entityToSerDe.computeIfAbsent( - entityClass, - k -> { - try { - Class> serdeClazz = - (Class>) - loadClass(ENTITY_TO_SERDE.get(k.getCanonicalName()), classLoader); - return serdeClazz.getDeclaredConstructor().newInstance(); - } catch (Exception e) { - throw new RuntimeException( - "Failed to instantiate serde class " + k.getCanonicalName(), e); - } - }); - } - - private Class getProtoClass( - Class entityClass, ClassLoader classLoader) throws IOException { - if (!ENTITY_TO_PROTO.containsKey(entityClass.getCanonicalName()) - || ENTITY_TO_PROTO.get(entityClass.getCanonicalName()) == null) { - throw new IOException("No proto class found for entity " + entityClass.getCanonicalName()); - } - return entityToProto.computeIfAbsent( - entityClass, - k -> { - try { - return (Class) - loadClass(ENTITY_TO_PROTO.get(k.getCanonicalName()), classLoader); - } catch (Exception e) { - throw new RuntimeException("Failed to create proto class " + k.getCanonicalName(), e); - } - }); - } - - private M toProto(T t, ClassLoader classLoader) - throws IOException { - ProtoSerDe protoSerDe = (ProtoSerDe) getProtoSerde(t.getClass(), classLoader); - return protoSerDe.serialize(t); - } - - private T fromProto( - M m, Class entityClass, ClassLoader classLoader, Namespace namespace) throws IOException { - ProtoSerDe protoSerDe = getProtoSerde(entityClass, classLoader); - return protoSerDe.deserialize(m, namespace); - } - - private Class loadClass(String className, ClassLoader classLoader) throws IOException { - try { - return Class.forName(className, true, classLoader); - } catch (Exception e) { - throw new IOException( - "Failed to load class " + className + " with classLoader " + classLoader, e); - } - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/ProtoSerDe.java b/core/src/main/java/org/apache/gravitino/proto/ProtoSerDe.java deleted file mode 100644 index 4081c02bd35..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/ProtoSerDe.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import com.google.protobuf.Message; -import org.apache.gravitino.Namespace; - -/** - * This interface defines the contract for a Protocol Buffer Serializer and Deserializer (SerDe). - * - * @param The entity type to be serialized and deserialized. - * @param The Protocol Buffer message type representing the entity. - */ -public interface ProtoSerDe { - - /** - * Serializes the provided entity into its corresponding Protocol Buffer message representation. - * - * @param t The entity to be serialized. - * @return The Protocol Buffer message representing the serialized entity. - */ - M serialize(T t); - - /** - * Deserializes the provided Protocol Buffer message into its corresponding entity representation. - * - * @param p The Protocol Buffer message to be deserialized. - * @param namespace The namespace to be specified for entity deserialization. - * @return The entity representing the deserialized Protocol Buffer message. - */ - T deserialize(M p, Namespace namespace); -} diff --git a/core/src/main/java/org/apache/gravitino/proto/ProtoSerDeException.java b/core/src/main/java/org/apache/gravitino/proto/ProtoSerDeException.java deleted file mode 100644 index e8dce6bf076..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/ProtoSerDeException.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -/** - * Custom exception class for Protocol Buffer Serializer and Deserializer (SerDe) related errors. - */ -public class ProtoSerDeException extends RuntimeException { - - /** - * Constructs a new ProtoSerDeException. - * - * @param message The error message. - */ - public ProtoSerDeException(String message) { - super(message); - } - - /** - * Constructs a new ProtoSerDeException. - * - * @param message The error message. - * @param cause The underlying cause of the exception. - */ - public ProtoSerDeException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/ProtoUtils.java b/core/src/main/java/org/apache/gravitino/proto/ProtoUtils.java deleted file mode 100644 index 877e1529892..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/ProtoUtils.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import com.google.protobuf.Timestamp; -import java.time.Instant; - -/** Utility class for converting between Java Instant and Protocol Buffer Timestamp. */ -public class ProtoUtils { - - private ProtoUtils() {} - - /** - * Converts a Java Instant object to a Protocol Buffer Timestamp object. - * - * @param instant The Java Instant to convert. - * @return The corresponding Protocol Buffer Timestamp. - */ - public static Timestamp fromInstant(Instant instant) { - return Timestamp.newBuilder() - .setSeconds(instant.getEpochSecond()) - .setNanos(instant.getNano()) - .build(); - } - - /** - * Converts a Protocol Buffer Timestamp object to a Java Instant object. - * - * @param timestamp The Protocol Buffer Timestamp to convert. - * @return The corresponding Java Instant. - */ - public static Instant toInstant(Timestamp timestamp) { - return Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java deleted file mode 100644 index a54f4505d7f..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import com.google.common.collect.Lists; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.gravitino.MetadataObject; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.authorization.Privilege; -import org.apache.gravitino.authorization.Privileges; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.meta.RoleEntity; - -public class RoleEntitySerDe implements ProtoSerDe { - - /** - * Serializes the provided entity into its corresponding Protocol Buffer message representation. - * - * @param roleEntity The entity to be serialized. - * @return The Protocol Buffer message representing the serialized entity. - */ - @Override - public Role serialize(RoleEntity roleEntity) { - Role.Builder builder = - Role.newBuilder() - .setId(roleEntity.id()) - .setName(roleEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())); - - for (SecurableObject securableObject : roleEntity.securableObjects()) { - builder.addSecurableObjects( - org.apache.gravitino.proto.SecurableObject.newBuilder() - .setFullName(securableObject.fullName()) - .setType(securableObject.type().name()) - .addAllPrivilegeConditions( - securableObject.privileges().stream() - .map(Privilege::condition) - .map(Privilege.Condition::name) - .collect(Collectors.toList())) - .addAllPrivilegeNames( - securableObject.privileges().stream() - .map(Privilege::name) - .map(Privilege.Name::name) - .collect(Collectors.toList())) - .build()); - } - - if (roleEntity.properties() != null && !roleEntity.properties().isEmpty()) { - builder.putAllProperties(roleEntity.properties()); - } - - return builder.build(); - } - - /** - * Deserializes the provided Protocol Buffer message into its corresponding entity representation. - * - * @param role The Protocol Buffer message to be deserialized. - * @return The entity representing the deserialized Protocol Buffer message. - */ - @Override - public RoleEntity deserialize(Role role, Namespace namespace) { - List securableObjects = Lists.newArrayList(); - - for (int index = 0; index < role.getSecurableObjectsCount(); index++) { - List privileges = Lists.newArrayList(); - org.apache.gravitino.proto.SecurableObject object = role.getSecurableObjects(index); - for (int privIndex = 0; privIndex < object.getPrivilegeConditionsCount(); privIndex++) { - if (Privilege.Condition.ALLOW.name().equals(object.getPrivilegeConditions(privIndex))) { - privileges.add(Privileges.allow(object.getPrivilegeNames(privIndex))); - } else { - privileges.add(Privileges.deny(object.getPrivilegeNames(privIndex))); - } - } - - SecurableObject securableObject = - SecurableObjects.parse( - object.getFullName(), MetadataObject.Type.valueOf(object.getType()), privileges); - - securableObjects.add(securableObject); - } - - RoleEntity.Builder builder = - RoleEntity.builder() - .withId(role.getId()) - .withName(role.getName()) - .withNamespace(namespace) - .withSecurableObjects(securableObjects) - .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); - - if (!role.getPropertiesMap().isEmpty()) { - builder.withProperties(role.getPropertiesMap()); - } - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/SchemaEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/SchemaEntitySerDe.java deleted file mode 100644 index d3515613f69..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/SchemaEntitySerDe.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.SchemaEntity; - -public class SchemaEntitySerDe implements ProtoSerDe { - @Override - public Schema serialize(SchemaEntity schemaEntity) { - Schema.Builder builder = - Schema.newBuilder() - .setId(schemaEntity.id()) - .setName(schemaEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(schemaEntity.auditInfo())); - - if (schemaEntity.comment() != null) { - builder.setComment(schemaEntity.comment()); - } - - if (schemaEntity.properties() != null && !schemaEntity.properties().isEmpty()) { - builder.putAllProperties(schemaEntity.properties()); - } - - return builder.build(); - } - - @Override - public SchemaEntity deserialize(Schema p, Namespace namespace) { - SchemaEntity.Builder builder = - SchemaEntity.builder() - .withId(p.getId()) - .withName(p.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); - - if (p.hasComment()) { - builder.withComment(p.getComment()); - } - - if (p.getPropertiesCount() > 0) { - builder.withProperties(p.getPropertiesMap()); - } - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/TableEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/TableEntitySerDe.java deleted file mode 100644 index 6c2b210d9f0..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/TableEntitySerDe.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.TableEntity; - -public class TableEntitySerDe implements ProtoSerDe { - @Override - public Table serialize(TableEntity tableEntity) { - return Table.newBuilder() - .setId(tableEntity.id()) - .setName(tableEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(tableEntity.auditInfo())) - .build(); - } - - @Override - public TableEntity deserialize(Table p, Namespace namespace) { - return TableEntity.builder() - .withId(p.getId()) - .withName(p.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)) - .build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/TopicEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/TopicEntitySerDe.java deleted file mode 100644 index 67ccd845e91..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/TopicEntitySerDe.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.TopicEntity; - -public class TopicEntitySerDe implements ProtoSerDe { - - @Override - public Topic serialize(TopicEntity topicEntity) { - Topic.Builder builder = - Topic.newBuilder() - .setId(topicEntity.id()) - .setName(topicEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(topicEntity.auditInfo())); - - if (topicEntity.comment() != null) { - builder.setComment(topicEntity.comment()); - } - - if (topicEntity.properties() != null && !topicEntity.properties().isEmpty()) { - builder.putAllProperties(topicEntity.properties()); - } - - return builder.build(); - } - - @Override - public TopicEntity deserialize(Topic p, Namespace namespace) { - TopicEntity.Builder builder = - TopicEntity.builder() - .withId(p.getId()) - .withName(p.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); - - if (p.hasComment()) { - builder.withComment(p.getComment()); - } - - if (p.getPropertiesCount() > 0) { - builder.withProperties(p.getPropertiesMap()); - } - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java deleted file mode 100644 index e859ee3b881..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import java.util.Collection; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.UserEntity; - -public class UserEntitySerDe implements ProtoSerDe { - - @Override - public User serialize(UserEntity userEntity) { - User.Builder builder = - User.newBuilder() - .setId(userEntity.id()) - .setName(userEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(userEntity.auditInfo())); - - if (isCollectionNotEmpty(userEntity.roles())) { - builder.addAllRoleNames(userEntity.roles()); - } - - if (isCollectionNotEmpty(userEntity.roleIds())) { - builder.addAllRoleIds(userEntity.roleIds()); - } - - return builder.build(); - } - - @Override - public UserEntity deserialize(User user, Namespace namespace) { - UserEntity.Builder builder = - UserEntity.builder() - .withId(user.getId()) - .withName(user.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo(), namespace)); - - if (user.getRoleNamesCount() > 0) { - builder.withRoleNames(user.getRoleNamesList()); - } - - if (user.getRoleIdsCount() > 0) { - builder.withRoleIds(user.getRoleIdsList()); - } - - return builder.build(); - } - - private boolean isCollectionNotEmpty(Collection collection) { - return collection != null && !collection.isEmpty(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/BiPredicate.java b/core/src/main/java/org/apache/gravitino/storage/BiPredicate.java deleted file mode 100644 index 036a23e95db..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/BiPredicate.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage; - -import java.io.IOException; - -/** - * Predicate for two parameters. It's a little like {@link java.util.function.Predicate}, but it - * throws {@link IOException} and has two parameters v1 and v2. - */ -@FunctionalInterface -public interface BiPredicate { - boolean test(T1 v1, T2 v2) throws IOException; -} diff --git a/core/src/main/java/org/apache/gravitino/storage/EntityKeyEncoder.java b/core/src/main/java/org/apache/gravitino/storage/EntityKeyEncoder.java deleted file mode 100644 index 8368ebe9ce3..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/EntityKeyEncoder.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage; - -import java.io.IOException; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.NameIdentifier; - -/** Interface for encoding entity to storage it in underlying storage. E.g., RocksDB. */ -public interface EntityKeyEncoder { - /** - * Construct the key for key-value store from the entity NameIdentifier and EntityType. - * - * @param ident entity identifier to encode - * @param type entity type to encode - * @return encoded key for key-value stored - * @throws IOException Exception if error occurs - */ - default T encode(NameIdentifier ident, EntityType type) throws IOException { - return encode(ident, type, false); - } - - /** - * Construct the key for key-value store from the entity NameIdentifier and EntityType. - * - * @param nullIfMissing return null if the specific entity no found - * @param type type of the ident that represents - * @param ident entity identifier to encode - * @return encoded key for key-value stored - * @throws IOException Exception if error occurs - */ - T encode(NameIdentifier ident, EntityType type, boolean nullIfMissing) throws IOException; - - /** - * Decode the key to NameIdentifier and EntityType. - * - * @param key the key to decode - * @return the pair of NameIdentifier and EntityType - * @throws IOException Exception if error occurs - */ - default Pair decode(T key) throws IOException { - throw new UnsupportedOperationException("Not implemented yet"); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/FunctionUtils.java b/core/src/main/java/org/apache/gravitino/storage/FunctionUtils.java deleted file mode 100644 index 3d1c6e01561..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/FunctionUtils.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage; - -import java.io.IOException; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.gravitino.storage.kv.TransactionalKvBackend; -import org.apache.gravitino.utils.Executable; - -/** Tools for executing functions with locks and transaction. */ -public class FunctionUtils { - - private FunctionUtils() {} - - @FunctionalInterface - public interface IOExecutable { - - R execute() throws IOException; - } - - public static R executeWithReadLock(IOExecutable executable, ReentrantReadWriteLock lock) - throws IOException { - // It already held the write lock - if (lock.isWriteLockedByCurrentThread()) { - return executable.execute(); - } - - lock.readLock().lock(); - try { - return executable.execute(); - } finally { - lock.readLock().unlock(); - } - } - - public static R executeWithWriteLock(IOExecutable executable, ReentrantReadWriteLock lock) - throws IOException { - lock.writeLock().lock(); - try { - return executable.execute(); - } finally { - lock.writeLock().unlock(); - } - } - - public static R executeInTransaction( - Executable executable, TransactionalKvBackend transactionalKvBackend) - throws E, IOException { - if (transactionalKvBackend.inTransaction()) { - return executable.execute(); - } - - transactionalKvBackend.begin(); - try { - R r = executable.execute(); - transactionalKvBackend.commit(); - return r; - } finally { - // Let GC do the roll-back work - transactionalKvBackend.closeTransaction(); - } - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/NameMappingService.java b/core/src/main/java/org/apache/gravitino/storage/NameMappingService.java deleted file mode 100644 index 6afdd43437d..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/NameMappingService.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage; - -import java.io.IOException; -import org.apache.gravitino.storage.kv.KvEntityStore; - -/** - * {@link NameMappingService} manages name to id mappings when using {@link KvEntityStore} to store - * entity. - * - *

Note. Implementations of this interface should be thread-safe. - */ -public interface NameMappingService extends AutoCloseable { - - /** - * Get id by name in the mapping service. - * - * @param name the name of the entity - * @return the id of the name, or null if the name does not exist - * @throws IOException if the underlying storage failed - */ - Long getIdByName(String name) throws IOException; - - /** - * Get name by id in the mapping service. - * - * @param id the id of the name - * @return the name of the id, or null if the id does not exist - * @throws IOException - */ - String getNameById(long id) throws IOException; - - /** - * Get id from name. If the name does not exist, we will create a new id for the name and bind the - * mapping between them - * - * @param name the name of the entity - * @return the id of the name - * @throws IOException if the underlying storage failed - */ - long getOrCreateIdFromName(String name) throws IOException; - - /** - * Update the mapping of the name to id. This method is used to update the mapping when we rename - * an entity. E.g., If we change the name of entity from oldName to nameName, the following is the - * mapping before and after the update. - * - * 

-   * Before:
-   *   oldName ------ 1
-   *   1       ------ oldName
-   *
-   * After:
-   *  newName ---- 1
-   *  1       ---- newName
-   *
- * - * @param oldName name to be updated - * @param newName new name - * @return true if the name exists and is updated successfully, false if the name does not exist - * @throws IOException if the underlying storage failed - */ - boolean updateName(String oldName, String newName) throws IOException; - - /** - * Unbind id-name mapping. Ignore if the name does not exist. - * - * @param name name to be unbind - * @return true if the name exists and is deleted successfully, false if the name does not exist - * @throws IOException if the underlying storage failed - */ - boolean unbindNameAndId(String name) throws IOException; -} diff --git a/core/src/main/java/org/apache/gravitino/storage/StorageLayoutException.java b/core/src/main/java/org/apache/gravitino/storage/StorageLayoutException.java deleted file mode 100644 index 09362f728ac..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/StorageLayoutException.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage; - -import com.google.errorprone.annotations.FormatMethod; -import com.google.errorprone.annotations.FormatString; -import org.apache.gravitino.exceptions.GravitinoRuntimeException; - -public class StorageLayoutException extends GravitinoRuntimeException { - - /** - * Constructs a new StorageLayoutException. - * - * @param message The detail message. - * @param args The argument of the formatted message. - */ - @FormatMethod - public StorageLayoutException(@FormatString String message, Object... args) { - super(message, args); - } - - /** - * Constructs a new StorageLayoutException. - * - * @param cause The cause of the exception. - * @param message The detail message. - * @param args The argument of the formatted message. - */ - @FormatMethod - public StorageLayoutException(Throwable cause, @FormatString String message, Object... args) { - super(cause, message, args); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/StorageLayoutVersion.java b/core/src/main/java/org/apache/gravitino/storage/StorageLayoutVersion.java deleted file mode 100644 index 88fc5d0afe4..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/StorageLayoutVersion.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage; - -import lombok.Getter; - -/** The storage layer version of the entity store. */ -@Getter -public enum StorageLayoutVersion { - V1("v1"); - - private final String version; - - StorageLayoutVersion(String version) { - this.version = version; - } - - public static StorageLayoutVersion fromString(String version) { - for (StorageLayoutVersion v : StorageLayoutVersion.values()) { - if (v.version.equals(version)) { - return v; - } - } - throw new StorageLayoutException( - "Unknown storage version, maybe the data is broken, please check the storage directory."); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/TransactionIdGenerator.java b/core/src/main/java/org/apache/gravitino/storage/TransactionIdGenerator.java deleted file mode 100644 index 45a6bf5c11e..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/TransactionIdGenerator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage; - -import java.io.Closeable; - -/** Generator id number as transaction, which will be used as MVCC version. */ -public interface TransactionIdGenerator extends Closeable { - - /** - * Next transaction id. The transaction id is a monotonically increasing number, which is used as - * MVCC version. - * - * @return next transaction id - */ - long nextId(); - - /** Start the generator. */ - void start(); -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityEncoderUtil.java b/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityEncoderUtil.java deleted file mode 100644 index a36b013de56..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityEncoderUtil.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Entity.EntityType.CATALOG; -import static org.apache.gravitino.Entity.EntityType.FILESET; -import static org.apache.gravitino.Entity.EntityType.METALAKE; -import static org.apache.gravitino.Entity.EntityType.SCHEMA; -import static org.apache.gravitino.Entity.EntityType.TABLE; -import static org.apache.gravitino.Entity.EntityType.TOPIC; -import static org.apache.gravitino.storage.kv.BinaryEntityKeyEncoder.LOG; -import static org.apache.gravitino.storage.kv.BinaryEntityKeyEncoder.NAMESPACE_SEPARATOR; -import static org.apache.gravitino.storage.kv.BinaryEntityKeyEncoder.TYPE_AND_NAME_SEPARATOR; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.storage.NameMappingService; - -public class BinaryEntityEncoderUtil { - - // The entity types in version 0.4.x, entities in this set do not have the prefix in the name-id - // mapping. Why do we introduce it? We need to make it backward compatible. - public static final Set VERSION_0_4_COMPATIBLE_ENTITY_TYPES = - ImmutableSet.of(METALAKE, CATALOG, SCHEMA, TABLE); - - private BinaryEntityEncoderUtil() {} - - /** - * Generate the key for name to id mapping. Currently, the mapping is as following. - * - * 
-   *   Assume we have the following entities:
-   *   metalake: a1        ---- 1
-   *   catalog : a1.b1     ---- 2
-   *   schema  : a1.b1.c   ---- 3
-   *
-   *   metalake: a2        ---- 4
-   *   catalog : a2.b2     ---- 5
-   *   schema  : a2.b2.c   ---- 6
-   *   schema  : a2.b2.c1  ---- 7
-   *
-   *   metalake: a1        ---- 1 means the name of metalake is a1 and the corresponding id is 1
-   *
- * - * Then we will store the name to id mapping as follows - * - * 
-   *  a1         -- 1
-   * 	1/b1       -- 2
-   * 	1/2/c      -- 3
-   * 	a2         -- 4
-   * 	4/b2       -- 5
-   * 	4/5/c      -- 6
-   * 	4/5/c1     -- 7
-   *
- * - * @param nameIdentifier name of a specific entity - * @return key that maps to the id of a specific entity. See above, The key maybe like '4/5/c1' - * @throws IOException if error occurs - */ - public static String generateKeyForMapping( - NameIdentifier nameIdentifier, EntityType entityType, NameMappingService nameMappingService) - throws IOException { - Namespace namespace = nameIdentifier.namespace(); - String name = nameIdentifier.name(); - - List parentTypes = EntityType.getParentEntityTypes(entityType); - long[] ids = new long[namespace.length()]; - for (int i = 0; i < ids.length; i++) { - ids[i] = - nameMappingService.getIdByName( - concatIdAndName( - ArrayUtils.subarray(ids, 0, i), namespace.level(i), parentTypes.get(i))); - } - - return concatIdAndName(ids, name, entityType); - } - - /** - * Concatenate the namespace ids and the name of the entity. Assuming the namespace ids are [1, 2] - * and the name is 'schema', the result will be '1/2/sc_schema'. - * - *

Attention, in order to make this change backward compatible, if the entity type is TABLE, we - * will not add a prefix to the name. If the entity type is not TABLE, we will add the prefix to - * the name. - * - * @param namespaceIds namespace ids, which are the ids of the parent entities - * @param name name of the entity - * @param type type of the entity - * @return concatenated string that used in the id-name mapping. - */ - public static String concatIdAndName(long[] namespaceIds, String name, EntityType type) { - String context = - Joiner.on(NAMESPACE_SEPARATOR) - .join( - Arrays.stream(namespaceIds).mapToObj(String::valueOf).collect(Collectors.toList())); - // We need to make it backward compatible, so we need to check if the name is already prefixed. - String mappingName = - VERSION_0_4_COMPATIBLE_ENTITY_TYPES.contains(type) - ? name - : type.getShortName() + TYPE_AND_NAME_SEPARATOR + name; - return StringUtils.isBlank(context) ? mappingName : context + NAMESPACE_SEPARATOR + mappingName; - } - - /** - * Get key prefix of all sub-entities under a specific entities. For example, as a metalake will - * start with `ml/{metalake_id}`, sub-entities under this metalake will have the prefix - * - * 

-   *   catalog: ca/{metalake_id}
-   *   schema:  sc/{metalake_id}
-   *   table:   ta/{metalake_id}
-   *
- * - * Why the sub-entities under this metalake start with those prefixes, please see {@link - * KvEntityStore} java class doc. - * - * @param ident identifier of an entity - * @param type type of entity - * @return list of sub-entities prefix - * @throws IOException if error occurs - */ - public static List getSubEntitiesPrefix( - NameIdentifier ident, EntityType type, BinaryEntityKeyEncoder entityKeyEncoder) - throws IOException { - List prefixes = Lists.newArrayList(); - byte[] encode = entityKeyEncoder.encode(ident, type, true); - switch (type) { - case METALAKE: - prefixes.add(replacePrefixTypeInfo(encode, CATALOG.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, SCHEMA.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, TABLE.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, FILESET.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, TOPIC.getShortName())); - break; - case CATALOG: - prefixes.add(replacePrefixTypeInfo(encode, SCHEMA.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, TABLE.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, FILESET.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, TOPIC.getShortName())); - break; - case SCHEMA: - prefixes.add(replacePrefixTypeInfo(encode, TABLE.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, FILESET.getShortName())); - prefixes.add(replacePrefixTypeInfo(encode, TOPIC.getShortName())); - break; - case TABLE: - case FILESET: - case TOPIC: - break; - default: - LOG.warn("Currently unknown type: {}, please check it", type); - } - Collections.reverse(prefixes); - return prefixes; - } - - /** - * Replace the prefix type info with the new type info. - * - * @param encode the encoded byte array - * @param subTypePrefix the new type prefix - * @return the new byte array - */ - public static byte[] replacePrefixTypeInfo(byte[] encode, String subTypePrefix) { - byte[] result = new byte[encode.length]; - System.arraycopy(encode, 0, result, 0, encode.length); - byte[] bytes = subTypePrefix.getBytes(StandardCharsets.UTF_8); - result[0] = bytes[0]; - result[1] = bytes[1]; - - return result; - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java b/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java deleted file mode 100644 index fa6fb48c636..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Entity.EntityType.CATALOG; -import static org.apache.gravitino.Entity.EntityType.FILESET; -import static org.apache.gravitino.Entity.EntityType.GROUP; -import static org.apache.gravitino.Entity.EntityType.METALAKE; -import static org.apache.gravitino.Entity.EntityType.ROLE; -import static org.apache.gravitino.Entity.EntityType.SCHEMA; -import static org.apache.gravitino.Entity.EntityType.TABLE; -import static org.apache.gravitino.Entity.EntityType.TOPIC; -import static org.apache.gravitino.Entity.EntityType.USER; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.storage.EntityKeyEncoder; -import org.apache.gravitino.storage.NameMappingService; -import org.apache.gravitino.utils.ByteUtils; -import org.apache.gravitino.utils.Bytes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Encode entity key for KV backend, e.g., RocksDB. The key is used to store the entity in the - * backend. The final key will be: - * - * 
- *     Key                                Value
- * ml/{ml_id}               -----    metalake info
- * ml/{ml_id}               -----    metalake info
- * ca/{ml_id}/{ca_id}       -----    catalog_info
- * ca/{ml_id}/{ca_id}       -----    catalog_info
- * sc/{ml_id}/{ca_id}/{sc_id} ---    schema_info
- * sc/{ml_id}/{ca_id}/{sc_id} ---    schema_info
- * br/{ml_id}/{ca_id}/{br_id} ---    broker_info
- * br/{ml_id}/{ca_id}/{br_id} ---    broker_info
- *
- * ta/{ml_id}/{ca_id}/{sc_id}/{table_id}    -----    table_info
- * ta/{ml_id}/{ca_id}/{sc_id}/{table_id}    -----    table_info
- * to/{ml_id}/{ca_id}/{br_id}/{to_id}       -----    topic_info
- * to/{ml_id}/{ca_id}/{br_id}/{to_id}       -----    topic_info
- *
- */ -public class BinaryEntityKeyEncoder implements EntityKeyEncoder { - public static final Logger LOG = LoggerFactory.getLogger(BinaryEntityKeyEncoder.class); - - public static final String NAMESPACE_SEPARATOR = "/"; - - public static final String TYPE_AND_NAME_SEPARATOR = "_"; - - @VisibleForTesting - static final byte[] BYTABLE_NAMESPACE_SEPARATOR = - NAMESPACE_SEPARATOR.getBytes(StandardCharsets.UTF_8); - - static final String WILD_CARD = "*"; - - // Key format template. Please the comment of the class for more details. - public static final Map ENTITY_TYPE_TO_NAME_IDENTIFIER = - ImmutableMap.of( - METALAKE, - new String[] {METALAKE.getShortName() + "/"}, - CATALOG, - new String[] {CATALOG.getShortName() + "/", "/"}, - SCHEMA, - new String[] {SCHEMA.getShortName() + "/", "/", "/"}, - TABLE, - new String[] {TABLE.getShortName() + "/", "/", "/", "/"}, - FILESET, - new String[] {FILESET.getShortName() + "/", "/", "/", "/"}, - USER, - new String[] {USER.getShortName() + "/", "/", "/", "/"}, - GROUP, - new String[] {GROUP.getShortName() + "/", "/", "/", "/"}, - ROLE, - new String[] {ROLE.getShortName() + "/", "/", "/", "/"}, - TOPIC, - new String[] {TOPIC.getShortName() + "/", "/", "/", "/"}); - - @VisibleForTesting final NameMappingService nameMappingService; - - public BinaryEntityKeyEncoder(NameMappingService nameMappingService) { - this.nameMappingService = nameMappingService; - } - - /** - * Encode entity key for KV backend, e.g., RocksDB. The key is used to store the entity in the - * backend. - * - * @param identifier NameIdentifier of the entity - * @param entityType the entity identifier to encode - * @param nullIfMissing return null if name-id mapping does not contain the mapping of identifier - * @return the encoded key for key-value storage. null if returnIfEntityNotFound is true and the - * entity the identifier represents does not exist; - */ - private byte[] encodeEntity( - NameIdentifier identifier, EntityType entityType, boolean nullIfMissing) throws IOException { - String[] nameSpace = identifier.namespace().levels(); - long[] namespaceIds = new long[nameSpace.length]; - List parentEntityTypes = EntityType.getParentEntityTypes(entityType); - for (int i = 0; i < nameSpace.length; i++) { - String nameKey = - BinaryEntityEncoderUtil.concatIdAndName( - ArrayUtils.subarray(namespaceIds, 0, i), nameSpace[i], parentEntityTypes.get(i)); - if (nullIfMissing && null == nameMappingService.getIdByName(nameKey)) { - return null; - } - - namespaceIds[i] = nameMappingService.getOrCreateIdFromName(nameKey); - } - - // If the name is a wildcard, We only need to encode the namespace. - if (WILD_CARD.equals(identifier.name())) { - String[] namespaceTemplate = ENTITY_TYPE_TO_NAME_IDENTIFIER.get(entityType); - if (namespaceTemplate == null) { - throw new UnsupportedOperationException("Unsupported entity type: " + entityType); - } - return formatNamespaceTemplateToByte(namespaceTemplate, namespaceIds); - } - - // This is for point query and need to use specific name - long[] namespaceAndNameIds = new long[namespaceIds.length + 1]; - System.arraycopy(namespaceIds, 0, namespaceAndNameIds, 0, namespaceIds.length); - String nameKey = - BinaryEntityEncoderUtil.concatIdAndName(namespaceIds, identifier.name(), entityType); - if (nullIfMissing && null == nameMappingService.getIdByName(nameKey)) { - return null; - } - - namespaceAndNameIds[namespaceIds.length] = nameMappingService.getOrCreateIdFromName(nameKey); - - String[] nameIdentifierTemplate = ENTITY_TYPE_TO_NAME_IDENTIFIER.get(entityType); - if (nameIdentifierTemplate == null) { - throw new UnsupportedOperationException("Unsupported entity type: " + entityType); - } - return formatNameIdentifierTemplateToByte(nameIdentifierTemplate, namespaceAndNameIds); - } - - /** - * Format the name space template to a byte array. For example, if the name space template is - * "ca/{}/" and the ids are [1], the result is "ca/1/" which means we want to get all catalogs in - * metalake '1' - * - * @param namespaceTemplate the name space template, please see {@link - * #ENTITY_TYPE_TO_NAME_IDENTIFIER} - * @param ids the ids that namespace names map to - */ - private byte[] formatNamespaceTemplateToByte(String[] namespaceTemplate, long[] ids) { - Preconditions.checkArgument(namespaceTemplate.length == ids.length + 1); - - byte[] bytes = new byte[0]; - for (int i = 0; i < namespaceTemplate.length; i++) { - if (i != namespaceTemplate.length - 1) { - bytes = - Bytes.concat( - bytes, - namespaceTemplate[i].getBytes(StandardCharsets.UTF_8), - ByteUtils.longToByte(ids[i])); - } else { - bytes = Bytes.concat(bytes, namespaceTemplate[i].getBytes(StandardCharsets.UTF_8)); - } - } - - return bytes; - } - - /** - * Format the name identifier to a byte array. For example, if the name space template is - * "ca/{}/{}" and the ids is [1, 2], the result is "ca/1/2" which means we want to get the - * specific catalog '2' - * - * @param nameIdentifierTemplate the name space template, please see {@link - * #ENTITY_TYPE_TO_NAME_IDENTIFIER} - * @param ids the ids that name identifier map to - */ - private byte[] formatNameIdentifierTemplateToByte(String[] nameIdentifierTemplate, long[] ids) { - Preconditions.checkArgument(nameIdentifierTemplate.length == ids.length); - - byte[] bytes = new byte[0]; - for (int i = 0; i < ids.length; i++) { - bytes = - Bytes.concat( - bytes, - nameIdentifierTemplate[i].getBytes(StandardCharsets.UTF_8), - ByteUtils.longToByte(ids[i])); - } - - return bytes; - } - - /** - * Encodes an entity object into a byte array for use as a key in a key-value store. - * - * @param ident NameIdentifier of the entity - * @param type the entity identifier to encode - * @param nullIfMissing return null if the entity with the name 'ident' not found - * @return The byte array representing the encoded key. - */ - @Override - public byte[] encode(NameIdentifier ident, EntityType type, boolean nullIfMissing) - throws IOException { - return encodeEntity(ident, type, nullIfMissing); - } - - /** - * Decodes a byte array into an entity object and corresponding entity type. - * - * @param key The byte array representing the encoded key. - * @return The decoded entity object and corresponding entity type. - * @throws IOException - */ - @Override - public Pair decode(byte[] key) throws IOException { - String entityTypeString = new String(ArrayUtils.subarray(key, 0, 2), StandardCharsets.UTF_8); - EntityType entityType = EntityType.fromShortName(entityTypeString); - - // If the entity type is a table, the key is encoded as: - // ta/{metalake_id}/{catalog_id}/{schema_id}/{table_id} and the length of the id is 8; - byte[] idArrays = ArrayUtils.subarray(key, 3, key.length); - long[] ids = new long[(idArrays.length + 1) / 9]; - int index = 0; - for (int i = 0; i < ids.length; i++) { - ids[i] = ByteUtils.byteToLong(ArrayUtils.subarray(idArrays, index, index + 8)); - index += 9; - } - - // Please review the id-name mapping content in KvNameMappingService.java and - // method generateMappingKey in this class. - String[] names = new String[ids.length]; - List parents = EntityType.getParentEntityTypes(entityType); - for (int i = 0; i < ids.length; i++) { - // The format of name is like '{metalake_id}/{catalog_id}/sc_schema_name' - String name = nameMappingService.getNameById(ids[i]); - // extract the real name from the name mapping service - // The name for table is 'table' NOT 'ta_table' to make it backward compatible. - EntityType currentEntityType = i < parents.size() ? parents.get(i) : entityType; - if (BinaryEntityEncoderUtil.VERSION_0_4_COMPATIBLE_ENTITY_TYPES.contains(currentEntityType)) { - names[i] = name.split(NAMESPACE_SEPARATOR, i + 1)[i]; - } else { - names[i] = name.split(NAMESPACE_SEPARATOR, i + 1)[i].substring(3); - } - } - - NameIdentifier nameIdentifier = NameIdentifier.of(names); - return Pair.of(nameIdentifier, entityType); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/KvBackend.java b/core/src/main/java/org/apache/gravitino/storage/kv/KvBackend.java deleted file mode 100644 index dd817ad9aeb..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/KvBackend.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.EntityAlreadyExistsException; - -/** Interface defining the operations for a Key-Value (KV) Backend. */ -public interface KvBackend extends Closeable { - /** - * Initializes the KV Backend environment with the provided configuration. - * - * @param config The configuration for the backend. - * @throws IOException If an I/O exception occurs during initialization. - */ - void initialize(Config config) throws IOException; - - /** - * Stores a key-value pair, possibly overwriting an existing value if specified. - * - * @param key The key of the pair. - * @param value The value of the pair. - * @param overwrite If true, overwrites the existing value. - * @throws IOException If an I/O exception occurs during the operation. - * @throws EntityAlreadyExistsException If the key already exists and overwrite is false. - */ - void put(byte[] key, byte[] value, boolean overwrite) - throws IOException, EntityAlreadyExistsException; - - /** - * Retrieves the value associated with a given key. - * - * @param key The key to retrieve the value for. - * @return The value associated with the key, or null if the key does not exist. - * @throws IOException If an I/O exception occurs during retrieval. - */ - byte[] get(byte[] key) throws IOException; - - /** - * Deletes the key-value pair associated with the given key. - * - * @param key The key to delete. - * @return True, if the key-value pair was successfully deleted, else throw exception. - * @throws IOException If an exception occurs during deletion. - */ - boolean delete(byte[] key) throws IOException; - - /** - * Delete the key-value pair associated with the given {@link KvRange} - * - * @param kvRange kv range to to delete - * @return True, if the key-value pair was successfully deleted, else throw exception. - * @throws IOException If an I/O exception occurs during deletion. - */ - boolean deleteRange(KvRange kvRange) throws IOException; - - /** - * Scans the specified range using the provided KvRangeScan and returns a list of key-value pairs. - * - * @param scanRange The range to scan. - * @return A list of key-value pairs within the specified range. - * @throws IOException If an I/O exception occurs during scanning. - */ - List> scan(KvRange scanRange) throws IOException; -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/kv/KvEntityStore.java deleted file mode 100644 index 20a9fb79eb0..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/KvEntityStore.java +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.ENTITY_KV_STORE; -import static org.apache.gravitino.Entity.EntityType.GROUP; -import static org.apache.gravitino.Entity.EntityType.METALAKE; -import static org.apache.gravitino.Entity.EntityType.ROLE; -import static org.apache.gravitino.Entity.EntityType.USER; -import static org.apache.gravitino.storage.kv.BinaryEntityEncoderUtil.generateKeyForMapping; -import static org.apache.gravitino.storage.kv.BinaryEntityEncoderUtil.getSubEntitiesPrefix; -import static org.apache.gravitino.storage.kv.BinaryEntityEncoderUtil.replacePrefixTypeInfo; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; -import lombok.Getter; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.Entity; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.EntityAlreadyExistsException; -import org.apache.gravitino.EntitySerDe; -import org.apache.gravitino.EntitySerDeFactory; -import org.apache.gravitino.EntityStore; -import org.apache.gravitino.HasIdentifier; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.exceptions.NoSuchEntityException; -import org.apache.gravitino.exceptions.NonEmptyEntityException; -import org.apache.gravitino.storage.EntityKeyEncoder; -import org.apache.gravitino.storage.FunctionUtils; -import org.apache.gravitino.storage.NameMappingService; -import org.apache.gravitino.storage.StorageLayoutVersion; -import org.apache.gravitino.storage.TransactionIdGenerator; -import org.apache.gravitino.utils.Bytes; -import org.apache.gravitino.utils.Executable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * KV store to store entities. This means we can store entities in a key value store. I.e., RocksDB, - * Cassandra, etc. If you want to use a different backend, you can implement the {@link KvBackend} - * interface - */ -public class KvEntityStore implements EntityStore { - - private static final String NO_SUCH_ENTITY_MSG = "No such entity:%s"; - - public static final Logger LOGGER = LoggerFactory.getLogger(KvEntityStore.class); - public static final ImmutableMap KV_BACKENDS = - ImmutableMap.of("RocksDBKvBackend", RocksDBKvBackend.class.getCanonicalName()); - public static final byte[] LAYOUT_VERSION_KEY = - Bytes.concat( - new byte[] {0x1D, 0x00, 0x02}, "layout_version".getBytes(StandardCharsets.UTF_8)); - - @Getter @VisibleForTesting KvBackend backend; - - // Lock to control the concurrency of the entity store, to be more exact, the concurrency of - // accessing the underlying kv store. - private ReentrantReadWriteLock reentrantReadWriteLock; - @VisibleForTesting EntityKeyEncoder entityKeyEncoder; - @VisibleForTesting NameMappingService nameMappingService; - private EntitySerDe serDe; - // We will use storageLayoutVersion to check whether the layout of the storage is compatible with - // the current version of the code. - // Note: If we change the layout of the storage in the future, please update the value of - // storageLayoutVersion if it's necessary. - @VisibleForTesting StorageLayoutVersion storageLayoutVersion; - - private TransactionIdGenerator txIdGenerator; - @VisibleForTesting KvGarbageCollector kvGarbageCollector; - private TransactionalKvBackend transactionalKvBackend; - - @Override - public void initialize(Config config) throws RuntimeException { - this.backend = createKvEntityBackend(config); - // TODO(yuqi) Currently, KvNameMappingService and KvEntityStore shares the same backend - // instance, We should make it configurable in the future. - this.txIdGenerator = new TransactionIdGeneratorImpl(backend, config); - txIdGenerator.start(); - - this.transactionalKvBackend = new TransactionalKvBackendImpl(backend, txIdGenerator); - - this.reentrantReadWriteLock = new ReentrantReadWriteLock(); - - this.nameMappingService = - new KvNameMappingService(transactionalKvBackend, reentrantReadWriteLock); - this.entityKeyEncoder = new BinaryEntityKeyEncoder(nameMappingService); - - this.kvGarbageCollector = new KvGarbageCollector(backend, config, entityKeyEncoder); - kvGarbageCollector.start(); - - this.storageLayoutVersion = initStorageVersionInfo(); - this.serDe = EntitySerDeFactory.createEntitySerDe(config); - } - - @Override - public void setSerDe(EntitySerDe entitySerDe) { - this.serDe = entitySerDe; - } - - @Override - public List list( - Namespace namespace, Class e, EntityType type) throws IOException { - // Star means it's a wildcard - List entities = Lists.newArrayList(); - NameIdentifier identifier = NameIdentifier.of(namespace, BinaryEntityKeyEncoder.WILD_CARD); - byte[] startKey = entityKeyEncoder.encode(identifier, type, true); - if (startKey == null) { - return entities; - } - - byte[] endKey = Bytes.increment(Bytes.wrap(startKey)).get(); - List> kvs = - executeInTransaction( - () -> - transactionalKvBackend.scan( - new KvRange.KvRangeBuilder() - .start(startKey) - .end(endKey) - .startInclusive(true) - .endInclusive(false) - .limit(Integer.MAX_VALUE) - .build())); - for (Pair pairs : kvs) { - entities.add(serDe.deserialize(pairs.getRight(), e, namespace)); - } - // TODO (yuqi), if the list is too large, we need to do pagination or streaming - return entities; - } - - @Override - public boolean exists(NameIdentifier ident, EntityType entityType) throws IOException { - return executeInTransaction( - () -> { - byte[] key = entityKeyEncoder.encode(ident, entityType, true); - if (key == null) { - return false; - } - return transactionalKvBackend.get(key) != null; - }); - } - - @Override - public void put(E e, boolean overwritten) - throws IOException, EntityAlreadyExistsException { - executeInTransaction( - () -> { - byte[] key = entityKeyEncoder.encode(e.nameIdentifier(), e.type()); - byte[] value = serDe.serialize(e); - transactionalKvBackend.put(key, value, overwritten); - return null; - }); - } - - @Override - public E update( - NameIdentifier ident, Class type, EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, EntityAlreadyExistsException { - return executeInTransaction( - () -> { - byte[] key = entityKeyEncoder.encode(ident, entityType); - byte[] value = transactionalKvBackend.get(key); - if (value == null) { - throw new NoSuchEntityException(NO_SUCH_ENTITY_MSG, ident.toString()); - } - - E e = serDe.deserialize(value, type, ident.namespace()); - E updatedE = updater.apply(e); - if (updatedE.nameIdentifier().equals(ident)) { - transactionalKvBackend.put(key, serDe.serialize(updatedE), true); - return updatedE; - } - - // If we have changed the name of the entity, We would do the following steps: - // Check whether the new entities already existed - boolean newEntityExist = exists(updatedE.nameIdentifier(), entityType); - if (newEntityExist) { - throw new EntityAlreadyExistsException( - "Entity %s already exist, please check again", updatedE.nameIdentifier()); - } - - // Update the name mapping - nameMappingService.updateName( - generateKeyForMapping(ident, entityType, nameMappingService), - generateKeyForMapping(updatedE.nameIdentifier(), entityType, nameMappingService)); - - // Update the entity to store - transactionalKvBackend.put(key, serDe.serialize(updatedE), true); - return updatedE; - }); - } - - @Override - public E get( - NameIdentifier ident, EntityType entityType, Class e) - throws NoSuchEntityException, IOException { - byte[] value = - executeInTransaction( - () -> { - byte[] key = entityKeyEncoder.encode(ident, entityType, true); - if (key == null) { - throw new NoSuchEntityException(NO_SUCH_ENTITY_MSG, ident.toString()); - } - return transactionalKvBackend.get(key); - }); - if (value == null) { - throw new NoSuchEntityException(NO_SUCH_ENTITY_MSG, ident.toString()); - } - return serDe.deserialize(value, e, ident.namespace()); - } - - void deleteAuthorizationEntitiesIfNecessary(NameIdentifier ident, EntityType type) - throws IOException { - if (type != METALAKE) { - return; - } - byte[] encode = entityKeyEncoder.encode(ident, type, true); - - String[] entityShortNames = - new String[] {USER.getShortName(), GROUP.getShortName(), ROLE.getShortName()}; - for (String name : entityShortNames) { - byte[] prefix = replacePrefixTypeInfo(encode, name); - transactionalKvBackend.deleteRange( - new KvRange.KvRangeBuilder() - .start(prefix) - .startInclusive(true) - .end(Bytes.increment(Bytes.wrap(prefix)).get()) - .build()); - } - } - - @Override - public boolean delete(NameIdentifier ident, EntityType entityType, boolean cascade) - throws IOException { - return executeInTransaction( - () -> { - if (!exists(ident, entityType)) { - return false; - } - - byte[] dataKey = entityKeyEncoder.encode(ident, entityType, true); - List subEntityPrefix = - getSubEntitiesPrefix(ident, entityType, (BinaryEntityKeyEncoder) entityKeyEncoder); - if (subEntityPrefix.isEmpty()) { - // has no sub-entities - return transactionalKvBackend.delete(dataKey); - } - - byte[] directChild = Iterables.getLast(subEntityPrefix); - byte[] endKey = Bytes.increment(Bytes.wrap(directChild)).get(); - List> kvs = - transactionalKvBackend.scan( - new KvRange.KvRangeBuilder() - .start(directChild) - .end(endKey) - .startInclusive(true) - .endInclusive(false) - .limit(1) - .build()); - - if (!cascade && !kvs.isEmpty()) { - List subEntities = Lists.newArrayListWithCapacity(kvs.size()); - for (Pair pair : kvs) { - subEntities.add(entityKeyEncoder.decode(pair.getLeft()).getLeft()); - } - - throw new NonEmptyEntityException( - "Entity %s has sub-entities %s, you should remove sub-entities first", - ident, subEntities); - } - - // Remove id-name mapping; - unbindNameAndId(ident, entityType); - - return transactionalKvBackend.delete(dataKey); - }); - } - - private void unbindNameAndId(NameIdentifier ident, EntityType entityType) throws IOException { - String identNameToIdKey = generateKeyForMapping(ident, entityType, nameMappingService); - nameMappingService.unbindNameAndId(identNameToIdKey); - } - - @Override - public R executeInTransaction(Executable executable) - throws E, IOException { - return FunctionUtils.executeInTransaction(executable, transactionalKvBackend); - } - - @Override - public void close() throws IOException { - txIdGenerator.close(); - kvGarbageCollector.close(); - backend.close(); - } - - private static KvBackend createKvEntityBackend(Config config) { - String backendName = config.get(ENTITY_KV_STORE); - String className = KV_BACKENDS.getOrDefault(backendName, backendName); - if (Objects.isNull(className)) { - throw new RuntimeException("Unsupported backend type..." + backendName); - } - - try { - KvBackend kvBackend = - (KvBackend) Class.forName(className).getDeclaredConstructor().newInstance(); - kvBackend.initialize(config); - return kvBackend; - } catch (Exception e) { - LOGGER.error("Failed to create and initialize KvBackend by name '{}'.", backendName, e); - throw new RuntimeException( - "Failed to create and initialize KvBackend by name: " + backendName, e); - } - } - - private StorageLayoutVersion initStorageVersionInfo() { - byte[] bytes; - try { - bytes = backend.get(LAYOUT_VERSION_KEY); - if (bytes == null) { - // If the layout version is not set, we will set it to the default version. - backend.put( - LAYOUT_VERSION_KEY, - StorageLayoutVersion.V1.getVersion().getBytes(StandardCharsets.UTF_8), - true); - return StorageLayoutVersion.V1; - } - - return StorageLayoutVersion.fromString(new String(bytes, StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new IllegalStateException("Failed to get/put layout version information", e); - } - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/KvGarbageCollector.java b/core/src/main/java/org/apache/gravitino/storage/kv/KvGarbageCollector.java deleted file mode 100644 index 55a82edd11f..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/KvGarbageCollector.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; -import static org.apache.gravitino.storage.kv.KvNameMappingService.GENERAL_NAME_MAPPING_PREFIX; -import static org.apache.gravitino.storage.kv.TransactionalKvBackendImpl.endOfTransactionId; -import static org.apache.gravitino.storage.kv.TransactionalKvBackendImpl.generateCommitKey; -import static org.apache.gravitino.storage.kv.TransactionalKvBackendImpl.generateKey; -import static org.apache.gravitino.storage.kv.TransactionalKvBackendImpl.getBinaryTransactionId; -import static org.apache.gravitino.storage.kv.TransactionalKvBackendImpl.getTransactionId; - -import com.google.common.annotations.VisibleForTesting; -import java.io.Closeable; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.SerializationUtils; -import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.storage.EntityKeyEncoder; -import org.apache.gravitino.utils.Bytes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * {@link KvGarbageCollector} is a garbage collector for the kv backend. It will collect the version - * of data which is not committed or exceed the ttl. - */ -public final class KvGarbageCollector implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(KvGarbageCollector.class); - private final KvBackend kvBackend; - private final Config config; - private final EntityKeyEncoder entityKeyEncoder; - private static final byte[] LAST_COLLECT_COMMIT_ID_KEY = - Bytes.concat( - new byte[] {0x1D, 0x00, 0x03}, "last_collect_commit_id".getBytes(StandardCharsets.UTF_8)); - - // Keep the last collect commit id to avoid collecting the same data multiple times, the first - // time the commit is 1 (minimum), and assuming we have collected the data with transaction id - // (1, 100], then the second time we collect the data and current tx_id is 200, - // then the current transaction id range is (100, 200] and so on. - byte[] commitIdHasBeenCollected; - private long frequencyInMinutes; - - private static final String TIME_STAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - - @VisibleForTesting - final ScheduledExecutorService garbageCollectorPool = - new ScheduledThreadPoolExecutor( - 2, - r -> { - Thread t = new Thread(r, "KvEntityStore-Garbage-Collector"); - t.setDaemon(true); - return t; - }, - new ThreadPoolExecutor.AbortPolicy()); - - public KvGarbageCollector( - KvBackend kvBackend, Config config, EntityKeyEncoder entityKeyEncoder) { - this.kvBackend = kvBackend; - this.config = config; - this.entityKeyEncoder = entityKeyEncoder; - } - - public void start() { - long dateTimelineMinute = config.get(STORE_DELETE_AFTER_TIME) / 1000 / 60; - - // We will collect garbage every 10 minutes at least. If the dateTimelineMinute is larger than - // 100 minutes, we would collect garbage every dateTimelineMinute/10 minutes. - this.frequencyInMinutes = Math.max(dateTimelineMinute / 10, 10); - garbageCollectorPool.scheduleAtFixedRate( - this::collectAndClean, 5, frequencyInMinutes, TimeUnit.MINUTES); - } - - @VisibleForTesting - void collectAndClean() { - LOG.info("Start to collect garbage..."); - try { - LOG.info("Start to collect and delete uncommitted data..."); - collectAndRemoveUncommittedData(); - - LOG.info("Start to collect and delete old version data..."); - collectAndRemoveOldVersionData(); - } catch (Exception e) { - LOG.error("Failed to collect garbage", e); - } - } - - private void collectAndRemoveUncommittedData() throws IOException { - List> kvs = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start(new byte[] {0x20}) // below 0x20 is control character - .end(new byte[] {0x7F}) // above 0x7F is control character - .startInclusive(true) - .endInclusive(false) - .predicate( - (k, v) -> { - byte[] transactionId = getBinaryTransactionId(k); - - // Only remove the uncommitted data that were written frequencyInMinutes - // minutes ago. - // It may have concurrency issues with TransactionalKvBackendImpl#commit. - long writeTime = getTransactionId(transactionId) >> 18; - if (writeTime - < (System.currentTimeMillis() - frequencyInMinutes * 60 * 1000 * 2)) { - return false; - } - - return kvBackend.get(generateCommitKey(transactionId)) == null; - }) - .limit(10000) /* Each time we only collect 10000 entities at most*/ - .build()); - - LOG.info("Start to remove {} uncommitted data", kvs.size()); - for (Pair pair : kvs) { - // Remove is a high-risk operation, So we log every delete operation - LogHelper logHelper = decodeKey(pair.getKey()); - LOG.info( - "Physically delete key that has marked uncommitted: name identity: '{}', entity type: '{}', createTime: '{}({})', key: '{}'", - logHelper.identifier, - logHelper.type, - logHelper.createTimeAsString, - logHelper.createTimeInMs, - pair.getKey()); - kvBackend.delete(pair.getKey()); - } - } - - private void collectAndRemoveOldVersionData() throws IOException { - long deleteTimeline = System.currentTimeMillis() - config.get(STORE_DELETE_AFTER_TIME); - // Why should we leave shift 18 bits? please refer to TransactionIdGeneratorImpl#nextId - // We can delete the data which is older than deleteTimeline.(old data with transaction id that - // is smaller than transactionIdToDelete) - long transactionIdToDelete = deleteTimeline << 18; - LOG.info("Start to remove data which is older than {}", transactionIdToDelete); - byte[] startKey = TransactionalKvBackendImpl.generateCommitKey(transactionIdToDelete); - commitIdHasBeenCollected = kvBackend.get(LAST_COLLECT_COMMIT_ID_KEY); - if (commitIdHasBeenCollected == null) { - commitIdHasBeenCollected = endOfTransactionId(); - } - - long lastGCId = getTransactionId(getBinaryTransactionId(commitIdHasBeenCollected)); - LOG.info( - "Start to collect data which is modified between '{}({})' (exclusive) and '{}({})' (inclusive)", - lastGCId, - lastGCId == 1 ? lastGCId : DateFormatUtils.format(lastGCId >> 18, TIME_STAMP_FORMAT), - transactionIdToDelete, - DateFormatUtils.format(deleteTimeline, TIME_STAMP_FORMAT)); - - // Get all commit marks - // TODO(yuqi), Use multi-thread to scan the data in case of the data is too large. - List> kvs = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start(startKey) - .end(commitIdHasBeenCollected) - .startInclusive(true) - .endInclusive(false) - .build()); - - for (Pair kv : kvs) { - List keysInTheTransaction = SerializationUtils.deserialize(kv.getValue()); - byte[] transactionId = getBinaryTransactionId(kv.getKey()); - - int keysDeletedCount = 0; - for (byte[] key : keysInTheTransaction) { - // Raw key format: {key} + {separator} + {transaction_id} - byte[] rawKey = generateKey(key, transactionId); - byte[] rawValue = kvBackend.get(rawKey); - if (null == rawValue) { - // It has been deleted - keysDeletedCount++; - continue; - } - - // Value has deleted mark, we can remove it. - if (null == TransactionalKvBackendImpl.getRealValue(rawValue)) { - // Delete the key of all versions. - removeAllVersionsOfKey(rawKey, key, false); - - LogHelper logHelper = decodeKey(key, transactionId); - kvBackend.delete(rawKey); - LOG.info( - "Physically delete key that has marked deleted: name identifier: '{}', entity type: '{}'," - + " createTime: '{}({})', key: '{}'", - logHelper.identifier, - logHelper.type, - logHelper.createTimeAsString, - logHelper.createTimeInMs, - Bytes.wrap(key)); - keysDeletedCount++; - continue; - } - - // If the key is not marked as deleted, then we need to check whether there is a newer - // version of the key. If there is a newer version of the key, then we can delete it - // directly. - List> newVersionOfKey = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start(key) - .end(generateKey(key, transactionId)) - .startInclusive(false) - .endInclusive(false) - .limit(1) - .build()); - if (!newVersionOfKey.isEmpty()) { - // Have a new version, we can safely remove all old versions. - removeAllVersionsOfKey(rawKey, key, false); - - // Has a newer version, we can remove it. - LogHelper logHelper = decodeKey(key, transactionId); - byte[] newVersionKey = newVersionOfKey.get(0).getKey(); - LogHelper newVersionLogHelper = decodeKey(newVersionKey); - kvBackend.delete(rawKey); - LOG.info( - "Physically delete key that has newer version: name identifier: '{}', entity type: '{}'," - + " createTime: '{}({})', newVersion createTime: '{}({})'," - + " key: '{}', newVersion key: '{}'", - logHelper.identifier, - logHelper.type, - logHelper.createTimeAsString, - logHelper.createTimeInMs, - newVersionLogHelper.createTimeAsString, - newVersionLogHelper.createTimeInMs, - Bytes.wrap(rawKey), - Bytes.wrap(newVersionKey)); - keysDeletedCount++; - } - } - - // All keys in this transaction have been deleted, we can remove the commit mark. - if (keysDeletedCount == keysInTheTransaction.size()) { - kvBackend.delete(kv.getKey()); - long timestamp = getTransactionId(transactionId) >> 18; - LOG.info( - "Physically delete commit mark: {}, createTime: '{}({})', key: '{}'", - Bytes.wrap(kv.getKey()), - DateFormatUtils.format(timestamp, TIME_STAMP_FORMAT), - timestamp, - Bytes.wrap(kv.getKey())); - } - } - - commitIdHasBeenCollected = kvs.isEmpty() ? startKey : kvs.get(0).getKey(); - kvBackend.put(LAST_COLLECT_COMMIT_ID_KEY, commitIdHasBeenCollected, true); - } - - /** - * Remove all versions of the key. - * - * @param rawKey raw key, it contains the transaction id. - * @param key key, it's the real key and does not contain the transaction id - * @param includeStart whether include the start key. - * @throws IOException if an I/O exception occurs during deletion. - */ - private void removeAllVersionsOfKey(byte[] rawKey, byte[] key, boolean includeStart) - throws IOException { - List> kvs = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start(rawKey) - .end(generateKey(key, 1)) - .startInclusive(includeStart) - .endInclusive(false) - .build()); - - for (Pair kv : kvs) { - // Delete real data. - kvBackend.delete(kv.getKey()); - - LogHelper logHelper = decodeKey(kv.getKey()); - LOG.info( - "Physically delete key that has marked deleted: name identifier: '{}', entity type: '{}'," - + " createTime: '{}({})', key: '{}'", - logHelper.identifier, - logHelper.type, - logHelper.createTimeAsString, - logHelper.createTimeInMs, - Bytes.wrap(key)); - - // Try to delete commit id if the all keys in the transaction id have been dropped. - byte[] transactionId = getBinaryTransactionId(kv.getKey()); - byte[] transactionKey = generateCommitKey(transactionId); - byte[] transactionValue = kvBackend.get(transactionKey); - - List keysInTheTransaction = SerializationUtils.deserialize(transactionValue); - - boolean allDropped = true; - for (byte[] keyInTheTransaction : keysInTheTransaction) { - if (kvBackend.get(generateKey(keyInTheTransaction, transactionId)) != null) { - // There is still a key in the transaction, we cannot delete the commit mark. - allDropped = false; - break; - } - } - - // Try to delete the commit mark. - if (allDropped) { - kvBackend.delete(transactionKey); - long timestamp = TransactionalKvBackendImpl.getTransactionId(transactionId) >> 18; - LOG.info( - "Physically delete commit mark: {}, createTime: '{}({})', key: '{}'", - Bytes.wrap(kv.getKey()), - DateFormatUtils.format(timestamp, TIME_STAMP_FORMAT), - timestamp, - Bytes.wrap(kv.getKey())); - } - } - } - - static class LogHelper { - - @VisibleForTesting final NameIdentifier identifier; - @VisibleForTesting final EntityType type; - @VisibleForTesting final long createTimeInMs; - // Formatted createTime - @VisibleForTesting final String createTimeAsString; - - public static final LogHelper NONE = new LogHelper(null, null, 0L, null); - - public LogHelper( - NameIdentifier identifier, - EntityType type, - long createTimeInMs, - String createTimeAsString) { - this.identifier = identifier; - this.type = type; - this.createTimeInMs = createTimeInMs; - this.createTimeAsString = createTimeAsString; - } - } - - @VisibleForTesting - LogHelper decodeKey(byte[] key, byte[] timestampArray) { - if (entityKeyEncoder == null) { - return LogHelper.NONE; - } - - // Name mapping data, we do not support it now. - if (Arrays.equals(GENERAL_NAME_MAPPING_PREFIX, ArrayUtils.subarray(key, 0, 3))) { - return LogHelper.NONE; - } - - Pair entityTypePair; - try { - entityTypePair = entityKeyEncoder.decode(key); - } catch (Exception e) { - LOG.warn("Unable to decode key: {}", Bytes.wrap(key), e); - return LogHelper.NONE; - } - long timestamp = getTransactionId(timestampArray) >> 18; - String ts = DateFormatUtils.format(timestamp, TIME_STAMP_FORMAT); - - return new LogHelper(entityTypePair.getKey(), entityTypePair.getValue(), timestamp, ts); - } - - @VisibleForTesting - LogHelper decodeKey(byte[] rawKey) { - byte[] key = TransactionalKvBackendImpl.getRealKey(rawKey); - byte[] timestampArray = TransactionalKvBackendImpl.getBinaryTransactionId(rawKey); - return decodeKey(key, timestampArray); - } - - @Override - public void close() throws IOException { - garbageCollectorPool.shutdownNow(); - try { - garbageCollectorPool.awaitTermination(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.error("Failed to close garbage collector", e); - } - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/KvNameMappingService.java b/core/src/main/java/org/apache/gravitino/storage/kv/KvNameMappingService.java deleted file mode 100644 index 05684a3732e..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/KvNameMappingService.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import com.google.common.annotations.VisibleForTesting; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.annotation.concurrent.ThreadSafe; -import org.apache.gravitino.storage.FunctionUtils; -import org.apache.gravitino.storage.IdGenerator; -import org.apache.gravitino.storage.NameMappingService; -import org.apache.gravitino.storage.RandomIdGenerator; -import org.apache.gravitino.utils.ByteUtils; -import org.apache.gravitino.utils.Bytes; - -/** - * {@link KvNameMappingService} is an implementation that use {@link KvBackend} to store name to id - * mapping. - */ -@ThreadSafe -public class KvNameMappingService implements NameMappingService { - - @VisibleForTesting final ReentrantReadWriteLock lock; - @VisibleForTesting final IdGenerator idGenerator = new RandomIdGenerator(); - - // To separate it from user keys, we will add three control flag 0x1D, 0x00, 0x00 as the prefix. - static final byte[] GENERAL_NAME_MAPPING_PREFIX = new byte[] {0x1D, 0x00, 0x00}; - - // Name prefix of name in name to id mapping, - // e.g., name_metalake1 -> 1 - // name_metalake2 -> 2 - private static final byte[] NAME_PREFIX = - Bytes.concat(GENERAL_NAME_MAPPING_PREFIX, "name_".getBytes(StandardCharsets.UTF_8)); - - // Id prefix of id in name to id mapping, - // e.g., id_1 -> metalake1 - // id_2 -> metalake2 - private static final byte[] ID_PREFIX = - Bytes.concat(GENERAL_NAME_MAPPING_PREFIX, "id_".getBytes(StandardCharsets.UTF_8)); - - @VisibleForTesting final TransactionalKvBackend transactionalKvBackend; - - public KvNameMappingService( - TransactionalKvBackend transactionalKvBackend, - ReentrantReadWriteLock reentrantReadWriteLock) { - this.transactionalKvBackend = transactionalKvBackend; - this.lock = reentrantReadWriteLock; - } - - @Override - public Long getIdByName(String name) throws IOException { - byte[] nameByte = getNameKey(name); - return FunctionUtils.executeInTransaction( - () -> { - byte[] idByte = transactionalKvBackend.get(nameByte); - return idByte == null ? null : ByteUtils.byteToLong(idByte); - }, - transactionalKvBackend); - } - - @Override - public String getNameById(long id) throws IOException { - byte[] idByte = getIdKey(id); - return FunctionUtils.executeInTransaction( - () -> { - byte[] name = transactionalKvBackend.get(idByte); - return name == null ? null : new String(name, StandardCharsets.UTF_8); - }, - transactionalKvBackend); - } - - private long bindNameAndId(String name) throws IOException { - byte[] nameByte = getNameKey(name); - long id = idGenerator.nextId(); - byte[] idByte = getIdKey(id); - return FunctionUtils.executeInTransaction( - () -> { - transactionalKvBackend.put(nameByte, ByteUtils.longToByte(id), false); - transactionalKvBackend.put(idByte, name.getBytes(StandardCharsets.UTF_8), false); - return id; - }, - transactionalKvBackend); - } - - @Override - public boolean updateName(String oldName, String newName) throws IOException { - return FunctionUtils.executeInTransaction( - () -> { - byte[] nameByte = getNameKey(oldName); - byte[] oldIdValue = transactionalKvBackend.get(nameByte); - // Old mapping has been deleted, no need to do it; - if (oldIdValue == null) { - return false; - } - - // Delete old name --> id mapping - transactionalKvBackend.delete(nameByte); - // In case there exists the mapping of new_name --> id, so we should use - // the overwritten strategy. In the following scenario, we should use the - // overwritten strategy: - // 1. Create name1 - // 2. Delete name1 - // 3. Create name2 - // 4. Rename name2 -> name1 - transactionalKvBackend.put(getNameKey(newName), oldIdValue, true); - transactionalKvBackend.put(oldIdValue, newName.getBytes(StandardCharsets.UTF_8), true); - return true; - }, - transactionalKvBackend); - } - - @Override - public boolean unbindNameAndId(String name) throws IOException { - byte[] nameByte = Bytes.concat(NAME_PREFIX, name.getBytes(StandardCharsets.UTF_8)); - return FunctionUtils.executeInTransaction( - () -> { - byte[] idByte = transactionalKvBackend.get(nameByte); - if (idByte == null) { - return false; - } - transactionalKvBackend.delete(nameByte); - transactionalKvBackend.delete(Bytes.concat(ID_PREFIX, idByte)); - return true; - }, - transactionalKvBackend); - } - - @Override - public void close() throws Exception {} - - @Override - public long getOrCreateIdFromName(String name) throws IOException { - Long id = getIdByName(name); - if (id == null) { - synchronized (this) { - if ((id = getIdByName(name)) == null) { - id = bindNameAndId(name); - } - } - } - - return id; - } - - /** Generate key to store id to name mapping. */ - private static byte[] getIdKey(long id) { - return Bytes.concat(ID_PREFIX, ByteUtils.longToByte(id)); - } - - /** Generate key to store name to id mapping. */ - private static byte[] getNameKey(String name) { - return Bytes.concat(NAME_PREFIX, name.getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/KvRange.java b/core/src/main/java/org/apache/gravitino/storage/kv/KvRange.java deleted file mode 100644 index 606ecbbc271..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/KvRange.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import com.google.common.base.Preconditions; -import lombok.Builder; -import lombok.Data; -import org.apache.gravitino.storage.BiPredicate; -import org.apache.gravitino.utils.Bytes; - -/** Represents a range scan/delete configuration for the key-value store. */ -@Builder -@Data -public class KvRange { - private byte[] start; - private byte[] end; - private boolean startInclusive; - private boolean endInclusive; - - private int limit; - private BiPredicate predicate; - - /** - * Constructs a KvRangeScan instance with the specified parameters. - * - * @param start The start key of the range. - * @param end The end key of the range. - * @param startInclusive True if the start key is inclusive, false otherwise. - * @param endInclusive True if the end key is inclusive, false otherwise. - * @param limit The maximum number of results to retrieve. - * @param predicate The predicate to use to filter key-value pairs. - */ - public KvRange( - byte[] start, - byte[] end, - boolean startInclusive, - boolean endInclusive, - int limit, - BiPredicate predicate) { - Preconditions.checkArgument(start != null, "start cannot be null"); - Preconditions.checkArgument(end != null, "start cannot be null"); - Preconditions.checkArgument( - Bytes.wrap(start).compareTo(end) <= 0, "start must be less than or equal to end"); - - if (limit == 0) { - limit = Integer.MAX_VALUE; - } - Preconditions.checkArgument(limit > 0, "limit must be greater than 0"); - - this.start = start; - this.end = end; - this.startInclusive = startInclusive; - this.endInclusive = endInclusive; - this.limit = limit; - this.predicate = predicate == null ? (k, v) -> true : predicate; - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/RocksDBKvBackend.java b/core/src/main/java/org/apache/gravitino/storage/kv/RocksDBKvBackend.java deleted file mode 100644 index 82e50e28097..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/RocksDBKvBackend.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.EntityAlreadyExistsException; -import org.apache.gravitino.utils.ByteUtils; -import org.apache.gravitino.utils.Bytes; -import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; -import org.rocksdb.TransactionDB; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * {@link RocksDBKvBackend} is a RocksDB implementation of KvBackend interface. If we want to use - * another kv implementation, We can just implement {@link KvBackend} interface and use it in the - * Gravitino. - */ -public class RocksDBKvBackend implements KvBackend { - public static final Logger LOGGER = LoggerFactory.getLogger(RocksDBKvBackend.class); - private RocksDB db; - - /** - * Initialize the RocksDB backend instance. We have used the {@link TransactionDB} to support - * transaction instead of {@link RocksDB} instance. - */ - private RocksDB initRocksDB(Config config) throws RocksDBException { - RocksDB.loadLibrary(); - - String dbPath = getStoragePath(config); - File dbDir = new File(dbPath, "instance"); - try (final Options options = new Options()) { - options.setCreateIfMissing(true); - - if (!dbDir.exists() && !dbDir.mkdirs()) { - throw new RocksDBException( - String.format("Can't create RocksDB path '%s'", dbDir.getAbsolutePath())); - } - LOGGER.info("Rocksdb storage directory:{}", dbDir); - // TODO (yuqi), make options configurable - return RocksDB.open(options, dbDir.getAbsolutePath()); - } catch (RocksDBException ex) { - LOGGER.error( - "Error initializing RocksDB, check configurations and permissions, exception: {}, message: {}, stackTrace: {}", - ex.getCause(), - ex.getMessage(), - ex.getStackTrace()); - throw ex; - } - } - - @VisibleForTesting - String getStoragePath(Config config) { - String dbPath = config.get(Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH); - if (StringUtils.isBlank(dbPath)) { - return Configs.DEFAULT_KV_ROCKSDB_BACKEND_PATH; - } - - Path path = Paths.get(dbPath); - // Relative Path - if (!path.isAbsolute()) { - path = Paths.get(System.getenv("GRAVITINO_HOME"), dbPath); - return path.toString(); - } - - return dbPath; - } - - @Override - public void initialize(Config config) throws IOException { - try { - db = initRocksDB(config); - } catch (RocksDBException e) { - throw new IOException(e); - } - } - - @Override - public void put(byte[] key, byte[] value, boolean overwrite) throws IOException { - try { - handlePut(key, value, overwrite); - } catch (EntityAlreadyExistsException e) { - throw e; - } catch (Exception e) { - throw new IOException(e); - } - } - - @VisibleForTesting - void handlePut(byte[] key, byte[] value, boolean overwrite) throws RocksDBException { - if (overwrite) { - db.put(key, value); - return; - } - byte[] existKey = db.get(key); - if (existKey != null) { - throw new EntityAlreadyExistsException( - "Key %s already exists in the database, please use overwrite option to overwrite it", - ByteUtils.formatByteArray(key)); - } - db.put(key, value); - } - - @Override - public byte[] get(byte[] key) throws IOException { - try { - return db.get(key); - } catch (RocksDBException e) { - throw new IOException(e); - } - } - - @Override - public List> scan(KvRange scanRange) throws IOException { - RocksIterator rocksIterator = db.newIterator(); - try { - rocksIterator.seek(scanRange.getStart()); - - List> result = Lists.newArrayList(); - int count = 0; - while (count < scanRange.getLimit() && rocksIterator.isValid()) { - byte[] key = rocksIterator.key(); - - // Break if the key is out of the scan range - if (Bytes.wrap(key).compareTo(scanRange.getEnd()) > 0) { - break; - } - - if (!scanRange.getPredicate().test(key, rocksIterator.value())) { - rocksIterator.next(); - continue; - } - - if (Bytes.wrap(key).compareTo(scanRange.getStart()) == 0) { - if (scanRange.isStartInclusive()) { - result.add(Pair.of(key, rocksIterator.value())); - count++; - } - } else if (Bytes.wrap(key).compareTo(scanRange.getEnd()) == 0) { - if (scanRange.isEndInclusive()) { - result.add(Pair.of(key, rocksIterator.value())); - } - break; - } else { - result.add(Pair.of(key, rocksIterator.value())); - count++; - } - - rocksIterator.next(); - } - return result; - } finally { - rocksIterator.close(); - } - } - - @Override - public boolean delete(byte[] key) throws IOException { - try { - db.delete(key); - return true; - } catch (RocksDBException e) { - throw new IOException(e); - } - } - - @Override - public boolean deleteRange(KvRange deleteRange) throws IOException { - RocksIterator rocksIterator = db.newIterator(); - - try { - rocksIterator.seek(deleteRange.getStart()); - - while (rocksIterator.isValid()) { - byte[] key = rocksIterator.key(); - // Break if the key is out of the scan range - if (Bytes.wrap(key).compareTo(deleteRange.getEnd()) > 0) { - break; - } - - if (Bytes.wrap(key).compareTo(deleteRange.getStart()) == 0) { - if (deleteRange.isStartInclusive()) { - delete(key); - } - } else if (Bytes.wrap(key).compareTo(deleteRange.getEnd()) == 0) { - if (deleteRange.isEndInclusive()) { - delete(key); - } - break; - } else { - delete(key); - } - - rocksIterator.next(); - } - return true; - } finally { - rocksIterator.close(); - } - } - - @Override - public void close() throws IOException { - db.close(); - } - - @VisibleForTesting - public RocksDB getDb() { - return db; - } - - @VisibleForTesting - public void setDb(RocksDB db) { - this.db = db; - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/TransactionIdGeneratorImpl.java b/core/src/main/java/org/apache/gravitino/storage/kv/TransactionIdGeneratorImpl.java deleted file mode 100644 index 897891ac4ab..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/TransactionIdGeneratorImpl.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.storage.TransactionIdGenerator; -import org.apache.gravitino.utils.ByteUtils; -import org.apache.gravitino.utils.Bytes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TransactionIdGeneratorImpl implements TransactionIdGenerator { - private static final Logger LOGGER = LoggerFactory.getLogger(TransactionIdGeneratorImpl.class); - - private final KvBackend kvBackend; - // We use three control characters 0x1D, 0x00, 0x01 to separate it from other keys. - private static final byte[] ID_GENERATOR_PREFIX = new byte[] {0x1D, 0x00, 0x01}; - - static final byte[] LAST_TIMESTAMP = - Bytes.concat(ID_GENERATOR_PREFIX, "last_timestamp".getBytes(StandardCharsets.UTF_8)); - - private volatile long incrementId = 0L; - private volatile long lastTransactionId = 0L; - private final Config config; - - private final ScheduledExecutorService scheduledThreadPoolExecutor = - new ScheduledThreadPoolExecutor( - 1, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("TransactionIdGenerator-thread-%d") - .setUncaughtExceptionHandler( - (t, e) -> LOGGER.error("Uncaught exception in thread {}", t, e)) - .build()); - - public TransactionIdGeneratorImpl(KvBackend kvBackend, Config config) { - this.kvBackend = kvBackend; - this.config = config; - } - - @Override - public void start() { - long maxSkewTime = config.get(Configs.STORE_TRANSACTION_MAX_SKEW_TIME); - // Why use maxSkewTime + 1? Because we will save the current timestamp to storage layer every - // maxSkewTime second and the save operation will also take a moment. Usually, it takes less - // than 1 millisecond, so we'd better wait maxSkewTime + 1000 millisecond to make sure the - // timestamp is OK. - checkTimeSkew(maxSkewTime + 1000); - - scheduledThreadPoolExecutor.scheduleAtFixedRate( - () -> { - int i = 0; - while (i++ < 3) { - try { - kvBackend.put(LAST_TIMESTAMP, ByteUtils.longToByte(System.currentTimeMillis()), true); - return; - } catch (IOException e) { - LOGGER.warn("Failed to save current timestamp to storage layer, retrying...", e); - } - } - - throw new RuntimeException( - "Failed to save current timestamp to storage layer after 3 retries, please check" - + "whether the storage layer is healthy."); - }, - maxSkewTime * 2, - maxSkewTime, - TimeUnit.MILLISECONDS); - } - - private void checkTimeSkew(long maxSkewTimeInMs) { - long current = System.currentTimeMillis(); - long old; - try { - old = getSavedTs(); - // In case of time skew, we will wait maxSkewTimeInSecond(default 2) seconds. - int retries = 0; - while (current <= old + maxSkewTimeInMs && retries++ < maxSkewTimeInMs / 100) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Thread was interrupted, exception: ", e); - } - current = System.currentTimeMillis(); - } - - if (current <= old + maxSkewTimeInMs) { - throw new RuntimeException( - String.format( - "Failed to initialize transaction id generator after %d milliseconds, time skew is too large", - maxSkewTimeInMs)); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Get the last saved timestamp from the storage layer. We will store the current timestamp to - * storage layer every 2(default) seconds. - */ - private long getSavedTs() throws IOException { - byte[] oldIdBytes = kvBackend.get(LAST_TIMESTAMP); - return oldIdBytes == null ? 0 : ByteUtils.byteToLong(oldIdBytes); - } - - /** - * We use the timestamp as the high 46 bits and the incrementId as the low 18 bits. The timestamp - * is always incremental. - */ - @Override - public synchronized long nextId() { - incrementId++; - if (incrementId >= ((1 << 18) - 1)) { - incrementId = 0; - } - - while (true) { - long tmpId = (System.currentTimeMillis() << 18) + incrementId; - if (tmpId > lastTransactionId) { - lastTransactionId = tmpId; - return tmpId; - } - } - } - - @Override - public void close() throws IOException { - scheduledThreadPoolExecutor.shutdownNow(); - try { - scheduledThreadPoolExecutor.awaitTermination(2000, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOGGER.warn( - "Failed to close thread pool scheduledThreadPoolExecutor in TransactionIdGeneratorImpl with in 2000 milliseconds", - e); - } - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackend.java b/core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackend.java deleted file mode 100644 index cf26fbee75c..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackend.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import java.io.IOException; - -public interface TransactionalKvBackend extends KvBackend { - - /** Begin the transaction. */ - void begin(); - - /** - * Commit the transaction. - * - * @throws IOException if the commit operation fails - */ - void commit() throws IOException; - - /** - * Rollback the transaction if something goes wrong. - * - * @throws IOException if the rollback operation fails - */ - void rollback() throws IOException; - - /** Close the current transaction. */ - void closeTransaction(); - - /** - * Check whether the backend is in transaction in the current thread. - * - * @return true if the backend is in transaction in the current thread - */ - boolean inTransaction(); -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackendImpl.java b/core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackendImpl.java deleted file mode 100644 index 8e3f44b129d..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/TransactionalKvBackendImpl.java +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import java.io.IOException; -import java.io.Serializable; -import java.util.List; -import javax.annotation.concurrent.ThreadSafe; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.SerializationUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.EntityAlreadyExistsException; -import org.apache.gravitino.storage.TransactionIdGenerator; -import org.apache.gravitino.utils.ByteUtils; -import org.apache.gravitino.utils.Bytes; - -/** - * TransactionalKvBackendImpl is an implementation of {@link TransactionalKvBackend} that uses 2PC - * (Two-Phase Commit) to support transaction. - * - *

Assuming we have a key-value pair (k1, v1) and a transaction id 1, the key-value pair will be - * store as - * - * 

- *       KEY                         VALUE
- *   key1 + separator + 1 --  status_code  + v1
- *   tx + separator + 1   --  binary that contains all keys involved in this tx
- *
- * - * We use '0x1F' as the separator, '______tx' as the value of tx, key1 + separator + 1 as the key of - * the value, tx + separator + 1 as the flag to indicate that the transaction 1 has been - * successfully committed and key1 can be visible or not, if transaction 1 fails(fail to write tx + - * separator + 1) and there is no tx + separator + 1, the key1 is not visible. - * - *

The status_code is a 20-byte integer that indicates the status of the value. The first four - * bytes of status code can be one of the following values: - * - * 

- *   0x00000000(Metrication: 0) -- NORMAL, the value is visible
- *   0x00000001(Metrication: 1) -- DELETED, the value is deleted and not visible
- *
- */ -@ThreadSafe -public class TransactionalKvBackendImpl implements TransactionalKvBackend { - private final KvBackend kvBackend; - private final TransactionIdGenerator transactionIdGenerator; - - @VisibleForTesting - final ThreadLocal>> putPairs = - ThreadLocal.withInitial(Lists::newArrayList); - - private final ThreadLocal> originalKeys = - ThreadLocal.withInitial(Lists::newArrayList); - - @VisibleForTesting final ThreadLocal txId = new ThreadLocal<>(); - - // 0x1E is control character RS - private static final byte[] TRANSACTION_PREFIX = {0x1E}; - - // Why use 8? We use 8 bytes to represent the status code, the first byte is used to - // identify the status of the value, the rest 7 bytes are for future use. - private static final int LENGTH_OF_VALUE_PREFIX = 8; - - // Why use 0x1F, 0x1F is a control character that is used as a delimiter in the text. - private static final byte[] SEPARATOR = new byte[] {0x1F}; - - private static final int LENGTH_OF_TRANSACTION_ID = Long.BYTES; - private static final int LENGTH_OF_SEPARATOR = SEPARATOR.length; - private static final int LENGTH_OF_VALUE_STATUS = Byte.BYTES; - - public TransactionalKvBackendImpl( - KvBackend kvBackend, TransactionIdGenerator transactionIdGenerator) { - this.kvBackend = kvBackend; - this.transactionIdGenerator = transactionIdGenerator; - } - - @Override - public void begin() { - if (!putPairs.get().isEmpty()) { - throw new IllegalStateException( - "The transaction is has not committed or rollback yet, you should commit or rollback it first"); - } - - txId.set(transactionIdGenerator.nextId()); - } - - @Override - public void commit() throws IOException { - try { - if (putPairs.get().isEmpty()) { - return; - } - - // Prepare - for (Pair pair : putPairs.get()) { - kvBackend.put(pair.getKey(), pair.getValue(), true); - } - - // Commit - kvBackend.put( - generateCommitKey(txId.get()), - SerializationUtils.serialize((Serializable) originalKeys.get()), - true); - } finally { - putPairs.get().clear(); - originalKeys.get().clear(); - txId.remove(); - } - } - - @Override - public void rollback() throws IOException { - // Delete the update value - for (Pair pair : putPairs.get()) { - kvBackend.delete(pair.getKey()); - } - } - - @Override - public void closeTransaction() { - putPairs.remove(); - originalKeys.remove(); - txId.remove(); - } - - @Override - public boolean inTransaction() { - return txId.get() != null; - } - - @Override - public void initialize(Config config) throws IOException {} - - @Override - public void put(byte[] key, byte[] value, boolean overwrite) - throws IOException, EntityAlreadyExistsException { - byte[] oldValue = get(key); - if (oldValue != null && !overwrite) { - throw new EntityAlreadyExistsException("Key already exists: %s", Bytes.wrap(key)); - } - putPairs - .get() - .add(Pair.of(generateKey(key, txId.get()), constructValue(value, ValueStatusEnum.NORMAL))); - originalKeys.get().add(key); - } - - @Override - public byte[] get(byte[] key) throws IOException { - byte[] rawValue = getNextReadableValue(key); - if (rawValue == null) { - return null; - } - - return getRealValue(rawValue); - } - - @Override - public boolean delete(byte[] key) throws IOException { - byte[] oldValue = get(key); - if (oldValue == null) { - return false; - } - - byte[] deletedValue = constructValue(oldValue, ValueStatusEnum.DELETED); - putPairs.get().add(Pair.of(generateKey(key, txId.get()), deletedValue)); - originalKeys.get().add(key); - return true; - } - - @Override - public boolean deleteRange(KvRange kvRange) throws IOException { - List> pairs = scan(kvRange); - pairs.forEach( - p -> - putPairs - .get() - .add( - Pair.of( - generateKey(p.getKey(), txId.get()), - constructValue(p.getValue(), ValueStatusEnum.DELETED)))); - return true; - } - - @Override - public List> scan(KvRange scanRange) throws IOException { - // Why we need to change the end key? Because we use the transaction id to construct a row key - // Assuming the end key is 'a' and the value of endInclusive is true, if we want to scan the - // value of key 'a', then we need to change the end key to 'b' and set the value of endInclusive - // to false. - byte[] end = scanRange.getEnd(); - boolean endInclude = scanRange.isEndInclusive(); - if (endInclude) { - end = endOfKey(end); - endInclude = false; - } - - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start(scanRange.getStart()) - .end(end) - .startInclusive(scanRange.isStartInclusive()) - .endInclusive(endInclude) - .predicate( - (k, v) -> { - byte[] transactionId = getBinaryTransactionId(k); - return kvBackend.get(generateCommitKey(transactionId)) != null; - }) - .limit(Integer.MAX_VALUE) - .build(); - - List> rawPairs = kvBackend.scan(kvRange); - List> result = Lists.newArrayList(); - int i = 0, j = 0; - while (i < scanRange.getLimit() && j < rawPairs.size()) { - Pair pair = rawPairs.get(j); - byte[] rawKey = pair.getKey(); - byte[] realKey = getRealKey(rawKey); - Bytes minNextKey = Bytes.increment(Bytes.wrap(Bytes.concat(realKey, SEPARATOR))); - - // If the start key is exclusive and the key is equal to the start key, we need to skip it. - if (!scanRange.isStartInclusive() - && Bytes.wrap(realKey).compareTo(scanRange.getStart()) == 0) { - while (j < rawPairs.size() && minNextKey.compareTo(rawPairs.get(j).getKey()) >= 0) { - j++; - } - continue; - } - - // If the end key is exclusive and the key is equal to the end key, we need to skip it. - if (!scanRange.isEndInclusive() && Bytes.wrap(realKey).compareTo(scanRange.getEnd()) == 0) { - break; - } - - byte[] value = getRealValue(pair.getValue()); - if (value != null) { - result.add(Pair.of(realKey, value)); - i++; - } - - j++; - // Skip all versions of the same key. - while (j < rawPairs.size() && minNextKey.compareTo(rawPairs.get(j).getKey()) >= 0) { - j++; - } - } - - return result; - } - - @Override - public void close() throws IOException {} - - public static byte[] getRealValue(byte[] rawValue) { - byte[] firstType = ArrayUtils.subarray(rawValue, 0, LENGTH_OF_VALUE_STATUS); - ValueStatusEnum statusEnum = ValueStatusEnum.fromCode(firstType[0]); - if (statusEnum == ValueStatusEnum.DELETED) { - // A deleted value is represented by a 4-byte integer with value 1 - return null; - } - return ArrayUtils.subarray(rawValue, LENGTH_OF_VALUE_PREFIX, rawValue.length); - } - - @VisibleForTesting - byte[] constructValue(byte[] value, ValueStatusEnum status) { - byte[] statusCode = new byte[] {status.getCode()}; - byte[] prefix = new byte[LENGTH_OF_VALUE_PREFIX]; - System.arraycopy(statusCode, 0, prefix, 0, statusCode.length); - return Bytes.concat(prefix, value); - } - - @VisibleForTesting - byte[] constructKey(byte[] key) { - return Bytes.concat(key, SEPARATOR, revertByteArray(ByteUtils.longToByte(txId.get()))); - } - - /** - * Get the latest readable value of the key as we support multi-version concurrency control - * mechanism to keep multiple version data. - */ - private byte[] getNextReadableValue(byte[] key) throws IOException { - List> pairs = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start(key) - .startInclusive(false) - .end(endOfKey(key)) - .endInclusive(false) - .predicate( - (k, v) -> { - byte[] transactionId = getBinaryTransactionId(k); - return kvBackend.get(generateCommitKey(transactionId)) != null; - }) - .limit(1) - .build()); - - if (pairs.isEmpty()) { - return null; - } - - return pairs.get(0).getValue(); - } - - /** - * Revert the bytes, Why we need to revert the bytes? Because we use the transaction id to - * construct a row key and need to place the latest version of the same key first. That is to say, - * the latest version of a key is the smallest one in alphabetical order, in this case, we would - * quickly locate the latest version of a key as key-value pair databases will sort keys in - * ascending order. - * - *

Let's say we have a key "key1" and the transaction id is 1, 2, 3, 4, 5, 6, 7, 8, 9, 10. The - * key-value pairs are: - * - * 

-   *   key1 10
-   *   key1 9
-   *   key1 8
-   *   ...
-   *
- * - * Assuming we have two long values, a and b, a >= b, then we always have: - * revert(ByteUtils.longToByte(a)) <= revert(ByteUtils.longToByte(b)) - * - *

When we try to get the value of key1, we will first the value of key1 10 and can skip old - * versions quickly. - */ - @VisibleForTesting - static byte[] revertByteArray(byte[] bytes) { - byte[] result = new byte[bytes.length]; - for (int i = 0; i < bytes.length; i++) { - result[i] = (byte) (bytes[i] ^ (byte) 0xff); - } - - return result; - } - - /** Generate a key of data for a specific transaction id. */ - static byte[] generateKey(byte[] key, long transactionId) { - return generateKey(key, revertByteArray(ByteUtils.longToByte(transactionId))); - } - - static byte[] generateKey(byte[] key, byte[] binaryTransactionId) { - return Bytes.concat(key, SEPARATOR, binaryTransactionId); - } - - /** Generate a commit key for a specific transaction id. */ - static byte[] generateCommitKey(long transactionId) { - byte[] binaryTransactionId = ByteUtils.longToByte(transactionId); - return generateCommitKey(revertByteArray(binaryTransactionId)); - } - - static byte[] generateCommitKey(byte[] transactionId) { - return Bytes.concat(TRANSACTION_PREFIX, SEPARATOR, transactionId); - } - - /** Get the end of transaction id, we use this key to scan all commit marks. */ - static byte[] endOfTransactionId() { - // Why use 1? Because we use 1 to represent the smallest transaction id. The smaller id will - // be larger than the bigger id when converting it to a byte array. - return generateCommitKey(1L); - } - - static byte[] endOfKey(byte[] key) { - return generateKey(key, 1L); - } - - static byte[] getRealKey(byte[] rawKey) { - return ArrayUtils.subarray( - rawKey, 0, rawKey.length - LENGTH_OF_TRANSACTION_ID - LENGTH_OF_SEPARATOR); - } - - /** Get the binary transaction id from the raw key. */ - static byte[] getBinaryTransactionId(byte[] rawKey) { - return ArrayUtils.subarray(rawKey, rawKey.length - LENGTH_OF_TRANSACTION_ID, rawKey.length); - } - - static long getTransactionId(byte[] binaryTransactionId) { - byte[] reverted = revertByteArray(binaryTransactionId); - return ByteUtils.byteToLong(reverted); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/ValueStatusEnum.java b/core/src/main/java/org/apache/gravitino/storage/kv/ValueStatusEnum.java deleted file mode 100644 index cc0002d96a7..00000000000 --- a/core/src/main/java/org/apache/gravitino/storage/kv/ValueStatusEnum.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -/** - * The status of a value. The value can be normal or deleted. The deleted value is not visible to - * the user and can be garbage collected. - * - *

In the future, we may add more status, such as tombstone and so on. - */ -public enum ValueStatusEnum { - // The value is normal. - NORMAL((byte) 0), - - // The value has been deleted. - DELETED((byte) 1); - - private final byte code; - - ValueStatusEnum(byte code) { - this.code = code; - } - - public byte getCode() { - return code; - } - - public static ValueStatusEnum fromCode(byte code) { - for (ValueStatusEnum valueStatusEnum : ValueStatusEnum.values()) { - if (valueStatusEnum.getCode() == code) { - return valueStatusEnum; - } - } - throw new IllegalArgumentException("Invalid code: " + code); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index a337e7a785e..a6d650c5470 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -28,7 +28,6 @@ import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; -import org.apache.gravitino.EntitySerDe; import org.apache.gravitino.EntityStore; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.MetadataObject; @@ -81,11 +80,6 @@ private static RelationalBackend createRelationalEntityBackend(Config config) { } } - @Override - public void setSerDe(EntitySerDe entitySerDe) { - throw new UnsupportedOperationException("Unsupported operation in relational entity store."); - } - @Override public List list( Namespace namespace, Class type, Entity.EntityType entityType) throws IOException { diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index c5d41adb008..30fa658130e 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -46,7 +46,6 @@ import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.TagEntity; import org.apache.gravitino.storage.IdGenerator; -import org.apache.gravitino.storage.kv.KvEntityStore; import org.apache.gravitino.utils.MetadataObjectUtil; import org.apache.gravitino.utils.PrincipalUtils; import org.slf4j.Logger; @@ -63,14 +62,6 @@ public class TagManager { private final SupportsTagOperations supportsTagOperations; public TagManager(IdGenerator idGenerator, EntityStore entityStore) { - if (entityStore instanceof KvEntityStore) { - String errorMsg = - "TagManager cannot run with kv entity store, please configure the entity " - + "store to use relational entity store and restart the Gravitino server"; - LOG.error(errorMsg); - throw new RuntimeException(errorMsg); - } - if (!(entityStore instanceof SupportsTagOperations)) { String errorMsg = "TagManager cannot run with entity store that does not support tag operations, " diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java index 9387fef0d5f..d0c2b1b2087 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -156,7 +156,6 @@ public static void setUp() throws Exception { entityStore = new TestMemoryEntityStore.InMemoryEntityStore(); entityStore.initialize(config); - entityStore.setSerDe(null); entityStore.put(metalakeEntity, true); entityStore.put(userEntity, true); diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java index 007ed7e84c9..d11bff572f1 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogManager.java @@ -80,7 +80,6 @@ public static void setUp() throws IOException { entityStore = new TestMemoryEntityStore.InMemoryEntityStore(); entityStore.initialize(config); - entityStore.setSerDe(null); entityStore.put(metalakeEntity, true); diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java index 72aa9af7693..66cf9874ca5 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestCatalogNormalizeDispatcher.java @@ -64,7 +64,6 @@ public static void setUp() throws IOException { entityStore = new TestMemoryEntityStore.InMemoryEntityStore(); entityStore.initialize(config); - entityStore.setSerDe(null); entityStore.put(metalakeEntity, true); diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java index 58cd8ce5f39..72415888c61 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestOperationDispatcher.java @@ -70,7 +70,6 @@ public static void setUp() throws IOException { entityStore = spy(new TestMemoryEntityStore.InMemoryEntityStore()); entityStore.initialize(config); - entityStore.setSerDe(null); BaseMetalake metalakeEntity = BaseMetalake.builder() diff --git a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java index 0d50c7fb767..cc2b9fd0d06 100644 --- a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java +++ b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java @@ -55,7 +55,6 @@ public static void setUp() { entityStore = new TestMemoryEntityStore.InMemoryEntityStore(); entityStore.initialize(config); - entityStore.setSerDe(null); metalakeManager = new MetalakeManager(entityStore, new RandomIdGenerator()); } diff --git a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeNormalizeDispatcher.java b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeNormalizeDispatcher.java index 1dac1d52650..2fbc2142bd9 100644 --- a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeNormalizeDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeNormalizeDispatcher.java @@ -42,7 +42,6 @@ public static void setUp() { entityStore = new TestMemoryEntityStore.InMemoryEntityStore(); entityStore.initialize(config); - entityStore.setSerDe(null); MetalakeManager metalakeManager = new MetalakeManager(entityStore, new RandomIdGenerator()); metalakeNormalizeDispatcher = new MetalakeNormalizeDispatcher(metalakeManager); diff --git a/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java deleted file mode 100644 index 1d4bfaa17d9..00000000000 --- a/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java +++ /dev/null @@ -1,432 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import java.io.IOException; -import java.time.Instant; -import java.util.Map; -import org.apache.gravitino.Catalog; -import org.apache.gravitino.Entity; -import org.apache.gravitino.EntitySerDe; -import org.apache.gravitino.EntitySerDeFactory; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.authorization.Privileges; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.meta.GroupEntity; -import org.apache.gravitino.meta.RoleEntity; -import org.apache.gravitino.meta.SchemaVersion; -import org.apache.gravitino.meta.UserEntity; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class TestEntityProtoSerDe { - - private final EntitySerDe entitySerDe = EntitySerDeFactory.createEntitySerDe("proto"); - - @Test - public void testAuditInfoSerDe() throws IOException { - Instant now = Instant.now(); - String creator = "creator"; - String modifier = "modifier"; - - org.apache.gravitino.meta.AuditInfo auditInfo = - org.apache.gravitino.meta.AuditInfo.builder() - .withCreator(creator) - .withCreateTime(now) - .withLastModifier(modifier) - .withLastModifiedTime(now) - .build(); - - ProtoEntitySerDe protoEntitySerDe = (ProtoEntitySerDe) entitySerDe; - - byte[] bytes = protoEntitySerDe.serialize(auditInfo); - org.apache.gravitino.meta.AuditInfo auditInfoFromBytes = - protoEntitySerDe.deserialize( - bytes, org.apache.gravitino.meta.AuditInfo.class, Namespace.empty()); - Assertions.assertEquals(auditInfo, auditInfoFromBytes); - - // Test with optional fields - org.apache.gravitino.meta.AuditInfo auditInfo1 = - org.apache.gravitino.meta.AuditInfo.builder() - .withCreator(creator) - .withCreateTime(now) - .build(); - - // Test from/to bytes - bytes = protoEntitySerDe.serialize(auditInfo1); - auditInfoFromBytes = - protoEntitySerDe.deserialize( - bytes, org.apache.gravitino.meta.AuditInfo.class, Namespace.empty()); - Assertions.assertEquals(auditInfo1, auditInfoFromBytes); - - // Test with empty field - org.apache.gravitino.meta.AuditInfo auditInfo2 = - org.apache.gravitino.meta.AuditInfo.builder().build(); - - byte[] bytes1 = protoEntitySerDe.serialize(auditInfo2); - org.apache.gravitino.meta.AuditInfo auditInfoFromBytes1 = - protoEntitySerDe.deserialize( - bytes1, org.apache.gravitino.meta.AuditInfo.class, Namespace.empty()); - Assertions.assertEquals(auditInfo2, auditInfoFromBytes1); - } - - @Test - public void testEntitiesSerDe() throws IOException { - Instant now = Instant.now(); - String creator = "creator"; - SchemaVersion version = SchemaVersion.V_0_1; - Long metalakeId = 1L; - String metalakeName = "metalake"; - Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); - - org.apache.gravitino.meta.AuditInfo auditInfo = - org.apache.gravitino.meta.AuditInfo.builder() - .withCreator(creator) - .withCreateTime(now) - .build(); - - // Test Metalake - org.apache.gravitino.meta.BaseMetalake metalake = - org.apache.gravitino.meta.BaseMetalake.builder() - .withId(metalakeId) - .withName(metalakeName) - .withProperties(props) - .withAuditInfo(auditInfo) - .withVersion(version) - .build(); - - ProtoEntitySerDe protoEntitySerDe = (ProtoEntitySerDe) entitySerDe; - - byte[] metalakeBytes = protoEntitySerDe.serialize(metalake); - org.apache.gravitino.meta.BaseMetalake metalakeFromBytes = - protoEntitySerDe.deserialize( - metalakeBytes, org.apache.gravitino.meta.BaseMetalake.class, Namespace.empty()); - Assertions.assertEquals(metalake, metalakeFromBytes); - - // Test metalake without props map - org.apache.gravitino.meta.BaseMetalake metalake1 = - org.apache.gravitino.meta.BaseMetalake.builder() - .withId(metalakeId) - .withName(metalakeName) - .withAuditInfo(auditInfo) - .withVersion(version) - .build(); - - byte[] metalakeBytes1 = protoEntitySerDe.serialize(metalake1); - org.apache.gravitino.meta.BaseMetalake metalakeFromBytes1 = - protoEntitySerDe.deserialize( - metalakeBytes1, org.apache.gravitino.meta.BaseMetalake.class, Namespace.empty()); - Assertions.assertEquals(metalake1, metalakeFromBytes1); - - // Test CatalogEntity - Long catalogId = 1L; - String catalogName = "catalog"; - String comment = "comment"; - String provider = "test"; - Namespace catalogNamespace = Namespace.of("metalake"); - - org.apache.gravitino.meta.CatalogEntity catalogEntity = - org.apache.gravitino.meta.CatalogEntity.builder() - .withId(catalogId) - .withName(catalogName) - .withNamespace(catalogNamespace) - .withComment(comment) - .withType(Catalog.Type.RELATIONAL) - .withProvider(provider) - .withAuditInfo(auditInfo) - .build(); - - byte[] catalogBytes = protoEntitySerDe.serialize(catalogEntity); - org.apache.gravitino.meta.CatalogEntity catalogEntityFromBytes = - protoEntitySerDe.deserialize( - catalogBytes, org.apache.gravitino.meta.CatalogEntity.class, catalogNamespace); - Assertions.assertEquals(catalogEntity, catalogEntityFromBytes); - - // Test Fileset catalog - org.apache.gravitino.meta.CatalogEntity filesetCatalogEntity = - org.apache.gravitino.meta.CatalogEntity.builder() - .withId(catalogId) - .withName(catalogName) - .withNamespace(catalogNamespace) - .withComment(comment) - .withType(Catalog.Type.FILESET) - .withProvider(provider) - .withAuditInfo(auditInfo) - .build(); - byte[] filesetCatalogBytes = protoEntitySerDe.serialize(filesetCatalogEntity); - org.apache.gravitino.meta.CatalogEntity filesetCatalogEntityFromBytes = - protoEntitySerDe.deserialize( - filesetCatalogBytes, org.apache.gravitino.meta.CatalogEntity.class, catalogNamespace); - Assertions.assertEquals(filesetCatalogEntity, filesetCatalogEntityFromBytes); - - // Test SchemaEntity - Namespace schemaNamespace = Namespace.of("metalake", "catalog"); - Long schemaId = 1L; - String schemaName = "schema"; - org.apache.gravitino.meta.SchemaEntity schemaEntity = - org.apache.gravitino.meta.SchemaEntity.builder() - .withId(schemaId) - .withName(schemaName) - .withNamespace(schemaNamespace) - .withAuditInfo(auditInfo) - .build(); - - byte[] schemaBytes = protoEntitySerDe.serialize(schemaEntity); - org.apache.gravitino.meta.SchemaEntity schemaEntityFromBytes = - protoEntitySerDe.deserialize( - schemaBytes, org.apache.gravitino.meta.SchemaEntity.class, schemaNamespace); - Assertions.assertEquals(schemaEntity, schemaEntityFromBytes); - - // Test SchemaEntity with additional fields - org.apache.gravitino.meta.SchemaEntity schemaEntity1 = - org.apache.gravitino.meta.SchemaEntity.builder() - .withId(schemaId) - .withName(schemaName) - .withNamespace(schemaNamespace) - .withAuditInfo(auditInfo) - .withComment(comment) - .withProperties(props) - .build(); - byte[] schemaBytes1 = protoEntitySerDe.serialize(schemaEntity1); - org.apache.gravitino.meta.SchemaEntity schemaEntityFromBytes1 = - protoEntitySerDe.deserialize( - schemaBytes1, org.apache.gravitino.meta.SchemaEntity.class, schemaNamespace); - Assertions.assertEquals(schemaEntity1, schemaEntityFromBytes1); - Assertions.assertEquals(comment, schemaEntityFromBytes1.comment()); - Assertions.assertEquals(props, schemaEntityFromBytes1.properties()); - - // Test TableEntity - Namespace tableNamespace = Namespace.of("metalake", "catalog", "schema"); - Long tableId = 1L; - String tableName = "table"; - org.apache.gravitino.meta.TableEntity tableEntity = - org.apache.gravitino.meta.TableEntity.builder() - .withId(tableId) - .withName(tableName) - .withNamespace(tableNamespace) - .withAuditInfo(auditInfo) - .build(); - - byte[] tableBytes = protoEntitySerDe.serialize(tableEntity); - org.apache.gravitino.meta.TableEntity tableEntityFromBytes = - protoEntitySerDe.deserialize( - tableBytes, org.apache.gravitino.meta.TableEntity.class, tableNamespace); - Assertions.assertEquals(tableEntity, tableEntityFromBytes); - - // Test FileEntity - Namespace filesetNamespace = Namespace.of("metalake", "catalog", "schema"); - Long fileId = 1L; - String fileName = "file"; - org.apache.gravitino.meta.FilesetEntity fileEntity = - org.apache.gravitino.meta.FilesetEntity.builder() - .withId(fileId) - .withName(fileName) - .withNamespace(filesetNamespace) - .withAuditInfo(auditInfo) - .withFilesetType(Fileset.Type.MANAGED) - .withStorageLocation("testLocation") - .withProperties(props) - .withComment(comment) - .build(); - byte[] fileBytes = protoEntitySerDe.serialize(fileEntity); - org.apache.gravitino.meta.FilesetEntity fileEntityFromBytes = - protoEntitySerDe.deserialize( - fileBytes, org.apache.gravitino.meta.FilesetEntity.class, filesetNamespace); - Assertions.assertEquals(fileEntity, fileEntityFromBytes); - - org.apache.gravitino.meta.FilesetEntity fileEntity1 = - org.apache.gravitino.meta.FilesetEntity.builder() - .withId(fileId) - .withName(fileName) - .withNamespace(filesetNamespace) - .withAuditInfo(auditInfo) - .withFilesetType(Fileset.Type.MANAGED) - .withStorageLocation("testLocation") - .build(); - byte[] fileBytes1 = protoEntitySerDe.serialize(fileEntity1); - org.apache.gravitino.meta.FilesetEntity fileEntityFromBytes1 = - protoEntitySerDe.deserialize( - fileBytes1, org.apache.gravitino.meta.FilesetEntity.class, filesetNamespace); - Assertions.assertEquals(fileEntity1, fileEntityFromBytes1); - Assertions.assertNull(fileEntityFromBytes1.comment()); - Assertions.assertNull(fileEntityFromBytes1.properties()); - - org.apache.gravitino.meta.FilesetEntity fileEntity2 = - org.apache.gravitino.meta.FilesetEntity.builder() - .withId(fileId) - .withName(fileName) - .withNamespace(filesetNamespace) - .withAuditInfo(auditInfo) - .withFilesetType(Fileset.Type.EXTERNAL) - .withProperties(props) - .withComment(comment) - .withStorageLocation("testLocation") - .build(); - byte[] fileBytes2 = protoEntitySerDe.serialize(fileEntity2); - org.apache.gravitino.meta.FilesetEntity fileEntityFromBytes2 = - protoEntitySerDe.deserialize( - fileBytes2, org.apache.gravitino.meta.FilesetEntity.class, filesetNamespace); - Assertions.assertEquals(fileEntity2, fileEntityFromBytes2); - Assertions.assertEquals("testLocation", fileEntityFromBytes2.storageLocation()); - Assertions.assertEquals(Fileset.Type.EXTERNAL, fileEntityFromBytes2.filesetType()); - - // Test TopicEntity - Namespace topicNamespace = Namespace.of("metalake", "catalog", "default"); - Long topicId = 1L; - String topicName = "topic"; - org.apache.gravitino.meta.TopicEntity topicEntity = - org.apache.gravitino.meta.TopicEntity.builder() - .withId(topicId) - .withName(topicName) - .withNamespace(topicNamespace) - .withAuditInfo(auditInfo) - .withComment(comment) - .withProperties(props) - .build(); - byte[] topicBytes = protoEntitySerDe.serialize(topicEntity); - org.apache.gravitino.meta.TopicEntity topicEntityFromBytes = - protoEntitySerDe.deserialize( - topicBytes, org.apache.gravitino.meta.TopicEntity.class, topicNamespace); - Assertions.assertEquals(topicEntity, topicEntityFromBytes); - - org.apache.gravitino.meta.TopicEntity topicEntity1 = - org.apache.gravitino.meta.TopicEntity.builder() - .withId(topicId) - .withName(topicName) - .withNamespace(topicNamespace) - .withAuditInfo(auditInfo) - .build(); - byte[] topicBytes1 = protoEntitySerDe.serialize(topicEntity1); - org.apache.gravitino.meta.TopicEntity topicEntityFromBytes1 = - protoEntitySerDe.deserialize( - topicBytes1, org.apache.gravitino.meta.TopicEntity.class, topicNamespace); - Assertions.assertEquals(topicEntity1, topicEntityFromBytes1); - Assertions.assertNull(topicEntityFromBytes1.comment()); - Assertions.assertNull(topicEntityFromBytes1.properties()); - - // Test UserEntity - Namespace userNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME); - Long userId = 1L; - String userName = "user"; - UserEntity userEntity = - UserEntity.builder() - .withId(userId) - .withName(userName) - .withNamespace(userNamespace) - .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) - .withRoleIds(Lists.newArrayList(1L)) - .build(); - byte[] userBytes = protoEntitySerDe.serialize(userEntity); - UserEntity userEntityFromBytes = - protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); - Assertions.assertEquals(userEntity, userEntityFromBytes); - - UserEntity userEntityWithoutFields = - UserEntity.builder() - .withId(userId) - .withName(userName) - .withNamespace(userNamespace) - .withAuditInfo(auditInfo) - .build(); - userBytes = protoEntitySerDe.serialize(userEntityWithoutFields); - userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); - Assertions.assertEquals(userEntityWithoutFields, userEntityFromBytes); - Assertions.assertNull(userEntityWithoutFields.roles()); - Assertions.assertNull(userEntityWithoutFields.roleIds()); - - // Test GroupEntity - Namespace groupNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME); - Long groupId = 1L; - String groupName = "group"; - - GroupEntity group = - GroupEntity.builder() - .withId(groupId) - .withName(groupName) - .withNamespace(groupNamespace) - .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) - .withRoleIds(Lists.newArrayList(1L)) - .build(); - byte[] groupBytes = protoEntitySerDe.serialize(group); - GroupEntity groupFromBytes = - protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); - Assertions.assertEquals(group, groupFromBytes); - - GroupEntity groupWithoutFields = - GroupEntity.builder() - .withId(groupId) - .withName(groupName) - .withNamespace(groupNamespace) - .withAuditInfo(auditInfo) - .build(); - groupBytes = protoEntitySerDe.serialize(groupWithoutFields); - groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); - Assertions.assertEquals(groupWithoutFields, groupFromBytes); - Assertions.assertNull(groupWithoutFields.roles()); - Assertions.assertNull(groupWithoutFields.roleIds()); - - // Test RoleEntity - Namespace roleNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME); - Long roleId = 1L; - String roleName = "testRole"; - String anotherCatalogName = "anotherCatalog"; - SecurableObject securableObject = - SecurableObjects.ofCatalog( - catalogName, - Lists.newArrayList(Privileges.UseCatalog.allow(), Privileges.CreateSchema.deny())); - SecurableObject anotherSecurableObject = - SecurableObjects.ofCatalog( - anotherCatalogName, Lists.newArrayList(Privileges.UseCatalog.allow())); - - RoleEntity roleEntity = - RoleEntity.builder() - .withId(roleId) - .withName(roleName) - .withNamespace(roleNamespace) - .withAuditInfo(auditInfo) - .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) - .withProperties(props) - .build(); - byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); - RoleEntity roleFromBytes = - protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); - Assertions.assertEquals(roleEntity, roleFromBytes); - - RoleEntity roleWithoutFields = - RoleEntity.builder() - .withId(1L) - .withName(roleName) - .withNamespace(roleNamespace) - .withAuditInfo(auditInfo) - .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) - .build(); - roleBytes = protoEntitySerDe.serialize(roleWithoutFields); - roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); - Assertions.assertEquals(roleWithoutFields, roleFromBytes); - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java index b0f516e50c8..fcc6ebd9b89 100644 --- a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java @@ -50,7 +50,6 @@ import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; @@ -103,9 +102,6 @@ public class TestEntityStorage { private static final Logger LOG = LoggerFactory.getLogger(TestEntityStorage.class); - public static final String KV_STORE_PATH = - "/tmp/gravitino_kv_entityStore_" + UUID.randomUUID().toString().replace("-", ""); - private static final String JDBC_STORE_PATH = "/tmp/gravitino_jdbc_entityStore_" + UUID.randomUUID().toString().replace("-", ""); private static final String DB_DIR = JDBC_STORE_PATH + "/testdb"; @@ -184,13 +180,7 @@ private void init(String type, Config config) { private void destroy(String type) { Preconditions.checkArgument(StringUtils.isNotBlank(type)); - if (type.equals(Configs.KV_STORE_KEY)) { - try { - FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH)); - } catch (Exception e) { - // Ignore - } - } else if (type.equalsIgnoreCase("h2") || type.equalsIgnoreCase("mysql")) { + if (type.equalsIgnoreCase("h2") || type.equalsIgnoreCase("mysql")) { dropAllTables(); File dir = new File(DB_DIR); if (dir.exists()) { @@ -2407,90 +2397,4 @@ private void validateDeletedColumns(Long entityId, Entity.EntityType entityType) listAllColumnWithEntityId(entityId, entityType); deleteResult.forEach(p -> Assertions.assertTrue(p.getRight().getRight() > 0)); } - - @ParameterizedTest - @MethodSource("storageProvider") - void testOptimizedDeleteForKv(String type) throws IOException { - if (!"kv".equalsIgnoreCase(type)) { - return; - } - - Config config = Mockito.mock(Config.class); - init(type, config); - - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - - BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo); - CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"), "catalog", auditInfo); - CatalogEntity catalogCopy = - createCatalog(2L, Namespace.of("metalake"), "catalogCopy", auditInfo); - - SchemaEntity schemaEntity = - createSchemaEntity(1L, Namespace.of("metalake", "catalog"), "schema1", auditInfo); - SchemaEntity schemaEntity2 = - createSchemaEntity(2L, Namespace.of("metalake", "catalog"), "schema2", auditInfo); - - TableEntity table = - createTableEntity( - 1L, Namespace.of("metalake", "catalog", "schema1"), "the same", auditInfo); - FilesetEntity filesetEntity = - createFilesetEntity( - 1L, Namespace.of("metalake", "catalog", "schema2"), "the same", auditInfo); - - store.put(metalake); - store.put(catalog); - store.put(catalogCopy); - store.put(schemaEntity); - store.put(schemaEntity2); - store.put(table); - store.put(filesetEntity); - - Assertions.assertDoesNotThrow( - () -> store.get(schemaEntity2.nameIdentifier(), EntityType.SCHEMA, SchemaEntity.class)); - Assertions.assertDoesNotThrow( - () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); - - // Test delete with the cascade or not - Assertions.assertThrows( - Exception.class, () -> store.delete(schemaEntity.nameIdentifier(), EntityType.SCHEMA)); - Assertions.assertDoesNotThrow( - () -> store.delete(schemaEntity.nameIdentifier(), EntityType.SCHEMA, true)); - - Assertions.assertDoesNotThrow( - () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); - - // Put the same schema back and see whether the deleted table exists or not - store.put(schemaEntity); - Assertions.assertDoesNotThrow( - () -> store.get(schemaEntity.nameIdentifier(), EntityType.SCHEMA, SchemaEntity.class)); - Assertions.assertThrows( - Exception.class, - () -> store.get(table.nameIdentifier(), EntityType.TABLE, TableEntity.class)); - Assertions.assertDoesNotThrow( - () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); - - store.put(table); - Assertions.assertDoesNotThrow( - () -> store.get(schemaEntity.nameIdentifier(), EntityType.SCHEMA, SchemaEntity.class)); - Assertions.assertDoesNotThrow( - () -> store.get(table.nameIdentifier(), EntityType.TABLE, TableEntity.class)); - Assertions.assertDoesNotThrow( - () -> store.get(filesetEntity.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); - - store.delete(table.nameIdentifier(), EntityType.TABLE); - FilesetEntity filesetEntity1 = - createFilesetEntity( - 1L, Namespace.of("metalake", "catalog", "schema1"), "the same", auditInfo); - store.put(filesetEntity1); - Assertions.assertDoesNotThrow( - () -> - store.get(filesetEntity1.nameIdentifier(), EntityType.FILESET, FilesetEntity.class)); - - destroy(type); - } - } } diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestEntityKeyEncoding.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestEntityKeyEncoding.java deleted file mode 100644 index dcf16f6884c..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestEntityKeyEncoding.java +++ /dev/null @@ -1,420 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_STORE; -import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; -import static org.apache.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; -import static org.apache.gravitino.storage.kv.BinaryEntityKeyEncoder.BYTABLE_NAMESPACE_SEPARATOR; -import static org.apache.gravitino.storage.kv.BinaryEntityKeyEncoder.WILD_CARD; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.Random; -import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.Entity.EntityType; -import org.apache.gravitino.EntitySerDeFactory; -import org.apache.gravitino.EntityStoreFactory; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.storage.IdGenerator; -import org.apache.gravitino.utils.ByteUtils; -import org.apache.gravitino.utils.Bytes; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mockito; - -@TestInstance(Lifecycle.PER_CLASS) -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -public class TestEntityKeyEncoding { - private Config getConfig() throws IOException { - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = java.nio.file.Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(3000L); - Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); - Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); - return config; - } - - private KvEntityStore getKvEntityStore(Config config) { - KvEntityStore kvEntityStore = (KvEntityStore) EntityStoreFactory.createEntityStore(config); - kvEntityStore.initialize(config); - kvEntityStore.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - return kvEntityStore; - } - - private IdGenerator getIdGeneratorAndSpy(BinaryEntityKeyEncoder entityKeyEncoder) - throws IllegalAccessException, NoSuchFieldException { - KvNameMappingService nameMappingService = - (KvNameMappingService) entityKeyEncoder.nameMappingService; - - Field field = nameMappingService.getClass().getDeclaredField("idGenerator"); - field.setAccessible(true); - IdGenerator idGenerator = (IdGenerator) field.get(nameMappingService); - IdGenerator spyIdGenerator = Mockito.spy(idGenerator); - field.set(nameMappingService, spyIdGenerator); - return spyIdGenerator; - } - - @Test - void testFilesetAndTableWithSameName() - throws IOException, NoSuchFieldException, IllegalAccessException { - Config config = getConfig(); - try (KvEntityStore kvEntityStore = getKvEntityStore(config)) { - BinaryEntityKeyEncoder encoder = (BinaryEntityKeyEncoder) kvEntityStore.entityKeyEncoder; - IdGenerator mockIdGenerator = getIdGeneratorAndSpy(encoder); - Mockito.doReturn(0L).when(mockIdGenerator).nextId(); - NameIdentifier mateLakeIdentifier1 = NameIdentifier.of("metalake"); - encoder.encode(mateLakeIdentifier1, EntityType.METALAKE); - - Mockito.doReturn(1L).when(mockIdGenerator).nextId(); - NameIdentifier catalogIdentifier = NameIdentifier.of("metalake", "catalogs"); - encoder.encode(catalogIdentifier, EntityType.CATALOG); - - Mockito.doReturn(2L).when(mockIdGenerator).nextId(); - NameIdentifier schemaIdentifier = NameIdentifier.of("metalake", "catalogs", "schema"); - encoder.encode(schemaIdentifier, EntityType.SCHEMA); - - Mockito.doReturn(3L).when(mockIdGenerator).nextId(); - NameIdentifier tableIdentifier = - NameIdentifier.of("metalake", "catalogs", "schema", "theSame"); - byte[] tableKey = encoder.encode(tableIdentifier, EntityType.TABLE); - byte[] expectKey = - Bytes.concat( - EntityType.TABLE.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(1L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(2L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(3L)); - - Assertions.assertArrayEquals(expectKey, tableKey); - - Mockito.doReturn(4L).when(mockIdGenerator).nextId(); - NameIdentifier filesetIdentifier = - NameIdentifier.of("metalake", "catalogs", "schema", "theSame"); - byte[] filesetKey = encoder.encode(filesetIdentifier, EntityType.FILESET); - - // Check the id of table is NOT the same as the id of fileset - Assertions.assertNotEquals(tableKey, filesetKey); - expectKey = - Bytes.concat( - EntityType.FILESET.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(1L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(2L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(4L)); - Assertions.assertArrayEquals(expectKey, filesetKey); - } - } - - @Test - public void testIdentifierEncoding() - throws IOException, IllegalAccessException, NoSuchFieldException { - Config config = getConfig(); - try (KvEntityStore kvEntityStore = getKvEntityStore(config)) { - BinaryEntityKeyEncoder encoder = (BinaryEntityKeyEncoder) kvEntityStore.entityKeyEncoder; - - // Metalake - // metalake1 --> 0 - Namespace namespace = Namespace.of(); - IdGenerator mockIdGenerator = getIdGeneratorAndSpy(encoder); - - Mockito.doReturn(0L).when(mockIdGenerator).nextId(); - NameIdentifier mateLakeIdentifier1 = NameIdentifier.of(namespace, "metalake1"); - byte[] realKey = encoder.encode(mateLakeIdentifier1, EntityType.METALAKE); - byte[] expectKey = - Bytes.concat( - EntityType.METALAKE.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L)); - Assertions.assertArrayEquals(expectKey, realKey); - - NameIdentifier decodeIdentifier = encoder.decode(realKey).getKey(); - Assertions.assertEquals(mateLakeIdentifier1, decodeIdentifier); - - // name ---> id - // catalog1 --> 1 - // catalog2 --> 2 - // catalog3 --> 3 - Namespace catalogNamespace = Namespace.of("metalake1"); - NameIdentifier catalogIdentifier1 = NameIdentifier.of(catalogNamespace, "catalog1"); - NameIdentifier catalogIdentifier2 = NameIdentifier.of(catalogNamespace, "catalog2"); - NameIdentifier catalogIdentifier3 = NameIdentifier.of(catalogNamespace, "catalog3"); - NameIdentifier[] catalogIdentifiers = - new NameIdentifier[] {catalogIdentifier1, catalogIdentifier2, catalogIdentifier3}; - - for (int i = 0; i < catalogIdentifiers.length; i++) { - Mockito.doReturn(1L + i).when(mockIdGenerator).nextId(); - NameIdentifier identifier = catalogIdentifiers[i]; - realKey = encoder.encode(identifier, EntityType.CATALOG); - expectKey = - Bytes.concat( - EntityType.CATALOG.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(1L + i)); - Assertions.assertArrayEquals(expectKey, realKey); - decodeIdentifier = encoder.decode(realKey).getKey(); - Assertions.assertEquals(identifier, decodeIdentifier); - } - - // name ---> id - // schema1 --> 4 - // schema2 --> 5 - // schema3 --> 6 - Namespace schemaNameSpace = Namespace.of("metalake1", "catalog2"); - NameIdentifier schemaIdentifier1 = NameIdentifier.of(schemaNameSpace, "schema1"); - NameIdentifier schemaIdentifier2 = NameIdentifier.of(schemaNameSpace, "schema2"); - NameIdentifier schemaIdentifier3 = NameIdentifier.of(schemaNameSpace, "schema3"); - NameIdentifier[] schemaIdentifiers = - new NameIdentifier[] {schemaIdentifier1, schemaIdentifier2, schemaIdentifier3}; - - for (int i = 0; i < schemaIdentifiers.length; i++) { - NameIdentifier identifier = schemaIdentifiers[i]; - Mockito.doReturn(4L + i).when(mockIdGenerator).nextId(); - realKey = encoder.encode(identifier, EntityType.SCHEMA); - expectKey = - Bytes.concat( - EntityType.SCHEMA.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(2L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(4L + i)); - Assertions.assertArrayEquals(expectKey, realKey); - decodeIdentifier = encoder.decode(realKey).getKey(); - Assertions.assertEquals(identifier, decodeIdentifier); - } - - // name ---> id - // table1 --> 7 - // table2 --> 8 - // table3 --> 9 - Namespace tableNameSpace = Namespace.of("metalake1", "catalog2", "schema3"); - NameIdentifier tableIdentifier1 = NameIdentifier.of(tableNameSpace, "table1"); - NameIdentifier tableIdentifier2 = NameIdentifier.of(tableNameSpace, "table2"); - NameIdentifier tableIdentifier3 = NameIdentifier.of(tableNameSpace, "table3"); - NameIdentifier[] tableIdentifiers = - new NameIdentifier[] {tableIdentifier1, tableIdentifier2, tableIdentifier3}; - - for (int i = 0; i < tableIdentifiers.length; i++) { - NameIdentifier identifier = tableIdentifiers[i]; - Mockito.doReturn(7L + i).when(mockIdGenerator).nextId(); - realKey = encoder.encode(identifier, EntityType.TABLE); - expectKey = - Bytes.concat( - EntityType.TABLE.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(2L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(6L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(i + 7L)); - Assertions.assertArrayEquals(expectKey, realKey); - decodeIdentifier = encoder.decode(realKey).getKey(); - Assertions.assertEquals(identifier, decodeIdentifier); - } - - // Unsupported operation - Mockito.doReturn(10L).when(mockIdGenerator).nextId(); - namespace = Namespace.of("metalake1", "catalog2", "schema3", "table1"); - NameIdentifier id = NameIdentifier.of(namespace, "column1"); - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> { - encoder.encode(id, EntityType.COLUMN); - }); - } - } - - @Test - public void testNamespaceEncoding() - throws IOException, IllegalAccessException, NoSuchFieldException { - Config config = getConfig(); - try (KvEntityStore kvEntityStore = getKvEntityStore(config)) { - BinaryEntityKeyEncoder encoder = (BinaryEntityKeyEncoder) kvEntityStore.entityKeyEncoder; - // Scan all Metalake - Namespace namespace = Namespace.of(); - IdGenerator mockIdGenerator = getIdGeneratorAndSpy(encoder); - - NameIdentifier metalakeIdentifier = NameIdentifier.of(namespace, WILD_CARD); - byte[] realKey = encoder.encode(metalakeIdentifier, EntityType.METALAKE); - byte[] expectKey = - Bytes.concat( - EntityType.METALAKE.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR); - Assertions.assertArrayEquals(expectKey, realKey); - - // Scan all catalog in metalake1 - // metalake1 --> 0L - Mockito.doReturn(0L).when(mockIdGenerator).nextId(); - Namespace catalogNamespace = Namespace.of("metalake1"); - NameIdentifier catalogIdentifier = NameIdentifier.of(catalogNamespace, WILD_CARD); - realKey = encoder.encode(catalogIdentifier, EntityType.CATALOG); - expectKey = - Bytes.concat( - EntityType.CATALOG.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR); - Assertions.assertArrayEquals(expectKey, realKey); - - // Scan all sc in metalake1.catalog2 - // catalog2 --> 1 - Mockito.doReturn(1L).when(mockIdGenerator).nextId(); - Namespace schemaNameSpace = Namespace.of("metalake1", "catalog2"); - NameIdentifier schemaIdentifier = NameIdentifier.of(schemaNameSpace, WILD_CARD); - realKey = encoder.encode(schemaIdentifier, EntityType.SCHEMA); - expectKey = - Bytes.concat( - EntityType.SCHEMA.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(1L), - BYTABLE_NAMESPACE_SEPARATOR); - Assertions.assertArrayEquals(expectKey, realKey); - - // Scan all table in metalake1.catalog2.schema3 - // schema3 --> 2 - Mockito.doReturn(2L).when(mockIdGenerator).nextId(); - Namespace tableNameSpace = Namespace.of("metalake1", "catalog2", "schema3"); - NameIdentifier tableIdentifier = NameIdentifier.of(tableNameSpace, WILD_CARD); - realKey = encoder.encode(tableIdentifier, EntityType.TABLE); - expectKey = - Bytes.concat( - EntityType.TABLE.getShortName().getBytes(StandardCharsets.UTF_8), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(0L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(1L), - BYTABLE_NAMESPACE_SEPARATOR, - ByteUtils.longToByte(2L), - BYTABLE_NAMESPACE_SEPARATOR); - Assertions.assertArrayEquals(expectKey, realKey); - - Mockito.doReturn(3L).when(mockIdGenerator).nextId(); - namespace = Namespace.of("metalake1", "catalog2", "schema3", "table1"); - NameIdentifier id = NameIdentifier.of(namespace, WILD_CARD); - Assertions.assertThrows( - UnsupportedOperationException.class, () -> encoder.encode(id, EntityType.COLUMN)); - } - } - - // Create random names for name identifiers. - private static Stream provideParameters() { - Random random = new Random(); - Arguments[] arguments = new Arguments[100]; - int identifierNameLen = 16; - - for (int i = 0; i < 100; i++) { - String[] value = new String[4]; - for (int j = 0; j < 4; j++) { - String tmp = ""; - int current = 0; - while (current < identifierNameLen) { - int v; - while ((v = random.nextInt(127)) < 32) {} - if (v % 4 == 0) { - v = 47; - } - - tmp += ((char) v); - current++; - } - - value[j] = tmp; - } - - arguments[i] = Arguments.of((Object[]) value); - } - - return Stream.of(arguments); - } - - @ParameterizedTest - @MethodSource("provideParameters") - void testSpecialCharacterDecoder( - String metalakeName, String catalogName, String schemaName, String tableName) - throws IOException { - Config config = getConfig(); - try (KvEntityStore kvEntityStore = getKvEntityStore(config)) { - BinaryEntityKeyEncoder encoder = (BinaryEntityKeyEncoder) kvEntityStore.entityKeyEncoder; - - NameIdentifier identifier = NameIdentifier.of(Namespace.of(), metalakeName); - byte[] key = encoder.encode(identifier, EntityType.METALAKE); - - Pair nameIdentifierEntityTypePair = encoder.decode(key); - Assertions.assertEquals(identifier, nameIdentifierEntityTypePair.getKey()); - Assertions.assertEquals(EntityType.METALAKE, nameIdentifierEntityTypePair.getValue()); - - identifier = NameIdentifier.of(Namespace.of(metalakeName), catalogName); - key = encoder.encode(identifier, EntityType.CATALOG); - nameIdentifierEntityTypePair = encoder.decode(key); - Assertions.assertEquals(identifier, nameIdentifierEntityTypePair.getKey()); - Assertions.assertEquals(EntityType.CATALOG, nameIdentifierEntityTypePair.getValue()); - - // Test Schema - identifier = NameIdentifier.of(Namespace.of(catalogName, catalogName), schemaName); - key = encoder.encode(identifier, EntityType.SCHEMA); - nameIdentifierEntityTypePair = encoder.decode(key); - Assertions.assertEquals(identifier, nameIdentifierEntityTypePair.getKey()); - Assertions.assertEquals(EntityType.SCHEMA, nameIdentifierEntityTypePair.getValue()); - - // Test Table - identifier = NameIdentifier.of(Namespace.of(catalogName, catalogName, schemaName), tableName); - key = encoder.encode(identifier, EntityType.TABLE); - nameIdentifierEntityTypePair = encoder.decode(key); - Assertions.assertEquals(identifier, nameIdentifierEntityTypePair.getKey()); - Assertions.assertEquals(EntityType.TABLE, nameIdentifierEntityTypePair.getValue()); - } - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestKvEntityStorage.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestKvEntityStorage.java deleted file mode 100644 index 75c3f30ba7c..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestKvEntityStorage.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_STORE; -import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; -import static org.apache.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.CompletionService; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import org.apache.commons.io.FileUtils; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.Entity; -import org.apache.gravitino.EntityAlreadyExistsException; -import org.apache.gravitino.EntitySerDeFactory; -import org.apache.gravitino.EntityStore; -import org.apache.gravitino.EntityStoreFactory; -import org.apache.gravitino.Metalake; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.exceptions.AlreadyExistsException; -import org.apache.gravitino.exceptions.NoSuchEntityException; -import org.apache.gravitino.exceptions.NonEmptyEntityException; -import org.apache.gravitino.meta.AuditInfo; -import org.apache.gravitino.meta.BaseMetalake; -import org.apache.gravitino.meta.CatalogEntity; -import org.apache.gravitino.storage.TestEntityStorage; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -public class TestKvEntityStorage extends TestEntityStorage { - @BeforeEach - @AfterEach - public void cleanEnv() { - try { - FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH)); - } catch (Exception e) { - // Ignore - } - } - - public Config getConfig() throws IOException { - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); - Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn("/tmp/gravitino"); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); - return config; - } - - @Test - void testCreateKvEntityStore() throws IOException { - Config config = getConfig(); - FileUtils.deleteDirectory(FileUtils.getFile("/tmp/gravitino")); - - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - Assertions.assertTrue(store instanceof KvEntityStore); - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - - BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo); - BaseMetalake metalakeCopy = createBaseMakeLake(2L, "metalakeCopy", auditInfo); - CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"), "catalog", auditInfo); - CatalogEntity catalogCopy = - createCatalog(2L, Namespace.of("metalake"), "catalogCopy", auditInfo); - CatalogEntity catalogCopyAgain = - createCatalog(3L, Namespace.of("metalake"), "catalogCopyAgain", auditInfo); - - // First, we try to test transactional is OK - final NameIdentifier metalakeID1 = metalake.nameIdentifier(); - Assertions.assertThrows( - NoSuchEntityException.class, - () -> store.get(metalakeID1, Entity.EntityType.METALAKE, BaseMetalake.class)); - - store.put(metalake); - store.put(catalog); - store.put(metalakeCopy); - store.put(catalogCopy); - store.put(catalogCopyAgain); - - Metalake retrievedMetalake = - store.get(metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); - Assertions.assertEquals(metalake, retrievedMetalake); - CatalogEntity retrievedCatalog = - store.get(catalog.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class); - Assertions.assertEquals(catalog, retrievedCatalog); - Metalake retrievedMetalakeCopy = - store.get(metalakeCopy.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); - Assertions.assertEquals(metalakeCopy, retrievedMetalakeCopy); - CatalogEntity retrievedCatalogCopy = - store.get(catalogCopy.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class); - Assertions.assertEquals(catalogCopy, retrievedCatalogCopy); - - // Test scan and store list interface - List catalogEntityList = - store.list(catalog.namespace(), CatalogEntity.class, Entity.EntityType.CATALOG); - Assertions.assertEquals(3, catalogEntityList.size()); - Assertions.assertTrue(catalogEntityList.contains(catalog)); - Assertions.assertTrue(catalogEntityList.contains(catalogCopy)); - Assertions.assertTrue(catalogEntityList.contains(catalogCopyAgain)); - - Assertions.assertThrows(EntityAlreadyExistsException.class, () -> store.put(catalog, false)); - store.delete(catalog.nameIdentifier(), Entity.EntityType.CATALOG); - final NameIdentifier metalakeID2 = catalog.nameIdentifier(); - Assertions.assertThrows( - NoSuchEntityException.class, - () -> store.get(metalakeID2, Entity.EntityType.CATALOG, CatalogEntity.class)); - - Assertions.assertThrows( - EntityAlreadyExistsException.class, () -> store.put(catalogCopy, false)); - store.delete(catalogCopy.nameIdentifier(), Entity.EntityType.CATALOG); - final NameIdentifier metalakeID3 = catalogCopy.nameIdentifier(); - Assertions.assertThrows( - NoSuchEntityException.class, - () -> store.get(metalakeID3, Entity.EntityType.CATALOG, CatalogEntity.class)); - - Assertions.assertThrowsExactly( - NonEmptyEntityException.class, - () -> store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); - store.delete(catalogCopyAgain.nameIdentifier(), Entity.EntityType.CATALOG); - Assertions.assertTrue(store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); - final NameIdentifier metalakeID4 = metalake.nameIdentifier(); - Assertions.assertThrows( - NoSuchEntityException.class, - () -> store.get(metalakeID4, Entity.EntityType.METALAKE, BaseMetalake.class)); - - // Test update - BaseMetalake updatedMetalake = createBaseMakeLake(1L, "updatedMetalake", auditInfo); - store.put(metalake); - store.update( - metalake.nameIdentifier(), - BaseMetalake.class, - Entity.EntityType.METALAKE, - l -> updatedMetalake); - Assertions.assertEquals( - updatedMetalake, - store.get( - updatedMetalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class)); - final NameIdentifier metalakeID5 = metalake.nameIdentifier(); - Assertions.assertThrows( - NoSuchEntityException.class, - () -> store.get(metalakeID5, Entity.EntityType.METALAKE, BaseMetalake.class)); - - // Add new updateMetaLake. - // 'updatedMetalake2' is a new name, which will trigger id allocation - BaseMetalake updatedMetalake2 = createBaseMakeLake(3L, "updatedMetalake2", auditInfo); - store.put(updatedMetalake2); - } - } - - @Test - @Disabled("KvEntityStore is not thread safe after issue #780") - void testConcurrentIssues() throws IOException, ExecutionException, InterruptedException { - Config config = getConfig(); - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - - ThreadPoolExecutor threadPoolExecutor = - new ThreadPoolExecutor( - 10, - 20, - 0L, - TimeUnit.MILLISECONDS, - new LinkedBlockingQueue(1000), - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("gravitino-t-%d").build()); - - CompletionService future = new ExecutorCompletionService<>(threadPoolExecutor); - - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - Assertions.assertTrue(store instanceof KvEntityStore); - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - - BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo); - CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"), "catalog", auditInfo); - - store.put(metalake); - store.put(catalog); - Assertions.assertNotNull( - store.get(catalog.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class)); - - // Delete the catalog entity, and we try to use multi-thread to delete it and make sure only - // one thread can delete it. - for (int i = 0; i < 10; i++) { - future.submit( - () -> - store.delete(NameIdentifier.of("metalake", "catalog"), Entity.EntityType.CATALOG)); - } - int totalSuccessNum = 0; - for (int i = 0; i < 10; i++) { - totalSuccessNum += future.take().get() ? 1 : 0; - } - Assertions.assertEquals(1, totalSuccessNum); - - // Try to use multi-thread to put the same catalog entity, and make sure only one thread can - // put it. - for (int i = 0; i < 20; i++) { - future.submit( - () -> { - store.put(catalog); /* overwrite is false, then only one will save it successfully */ - return null; - }); - } - - int totalFailed = 0; - for (int i = 0; i < 20; i++) { - try { - future.take().get(); - } catch (Exception e) { - Assertions.assertTrue(e.getCause() instanceof EntityAlreadyExistsException); - totalFailed++; - } - } - Assertions.assertEquals(19, totalFailed); - - // Try to use multi-thread to update the same catalog entity, and make sure only one thread - // can update it. - for (int i = 0; i < 10; i++) { - future.submit( - () -> { - // Ten threads rename the catalog entity from 'catalog' to 'catalog1' at the same - // time. - store.update( - NameIdentifier.of("metalake", "catalog"), - CatalogEntity.class, - Entity.EntityType.CATALOG, - e -> { - AuditInfo auditInfo1 = - AuditInfo.builder() - .withCreator("creator1") - .withCreateTime(Instant.now()) - .build(); - return createCatalog(1L, Namespace.of("metalake"), "catalog1", auditInfo1); - }); - return null; - }); - } - - totalFailed = 0; - for (int i = 0; i < 10; i++) { - try { - future.take().get(); - } catch (Exception e) { - // It may throw NoSuchEntityException or AlreadyExistsException - // NoSuchEntityException: because old entity has been renamed by the other thread already, - // we can't get the old one. - // AlreadyExistsException: because the entity has been renamed by the other thread - // already, we can't rename it again. - Assertions.assertTrue( - e.getCause() instanceof AlreadyExistsException - || e.getCause() instanceof NoSuchEntityException); - totalFailed++; - } - } - Assertions.assertEquals(9, totalFailed); - } - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestKvGarbageCollector.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestKvGarbageCollector.java deleted file mode 100644 index 457ecdae9ba..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestKvGarbageCollector.java +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_STORE; -import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; -import static org.apache.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; -import static org.apache.gravitino.storage.kv.TestKvEntityStorage.createBaseMakeLake; -import static org.apache.gravitino.storage.kv.TestKvEntityStorage.createCatalog; -import static org.apache.gravitino.storage.kv.TestKvEntityStorage.createFilesetEntity; -import static org.apache.gravitino.storage.kv.TestKvEntityStorage.createSchemaEntity; -import static org.apache.gravitino.storage.kv.TestKvEntityStorage.createTableEntity; -import static org.apache.gravitino.storage.kv.TransactionalKvBackendImpl.getBinaryTransactionId; -import static org.apache.gravitino.storage.kv.TransactionalKvBackendImpl.getTransactionId; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.time.Instant; -import java.util.List; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.Entity; -import org.apache.gravitino.EntitySerDeFactory; -import org.apache.gravitino.EntityStore; -import org.apache.gravitino.EntityStoreFactory; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.AuditInfo; -import org.apache.gravitino.meta.BaseMetalake; -import org.apache.gravitino.meta.CatalogEntity; -import org.apache.gravitino.meta.FilesetEntity; -import org.apache.gravitino.meta.GroupEntity; -import org.apache.gravitino.meta.SchemaEntity; -import org.apache.gravitino.meta.TableEntity; -import org.apache.gravitino.meta.UserEntity; -import org.apache.gravitino.storage.TransactionIdGenerator; -import org.apache.gravitino.storage.kv.KvGarbageCollector.LogHelper; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -@SuppressWarnings("DefaultCharset") -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -class TestKvGarbageCollector { - public Config getConfig() throws IOException { - Config config = Mockito.mock(Config.class); - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); - Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(3L); - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); - return config; - } - - private KvBackend getKvBackEnd(Config config) throws IOException { - KvBackend kvBackend = new RocksDBKvBackend(); - kvBackend.initialize(config); - return kvBackend; - } - - @Test - void testScheduler() throws IOException { - Config config = getConfig(); - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); // 20 minutes - long dateTimelineMinute = config.get(STORE_DELETE_AFTER_TIME) / 1000 / 60; - Assertions.assertEquals(10, Math.max(dateTimelineMinute / 10, 10)); - - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(2 * 60 * 60 * 1000L); // 2 hours - dateTimelineMinute = config.get(STORE_DELETE_AFTER_TIME) / 1000 / 60; - Assertions.assertEquals(12, Math.max(dateTimelineMinute / 10, 10)); - - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)) - .thenReturn(2 * 60 * 60 * 24 * 1000L); // 2 days - dateTimelineMinute = config.get(STORE_DELETE_AFTER_TIME) / 1000 / 60; - Assertions.assertEquals(288, Math.max(dateTimelineMinute / 10, 10)); - } - - @Test - void testCollectGarbage() throws IOException, InterruptedException { - Config config = getConfig(); - try (KvBackend kvBackend = getKvBackEnd(config)) { - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackendImpl transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - transactionalKvBackend.begin(); - transactionalKvBackend.put("testA".getBytes(), "v1".getBytes(), true); - transactionalKvBackend.put("testB".getBytes(), "v1".getBytes(), true); - transactionalKvBackend.put("testC".getBytes(), "v1".getBytes(), true); - transactionalKvBackend.commit(); - transactionalKvBackend.closeTransaction(); - - transactionalKvBackend.begin(); - transactionalKvBackend.put("testA".getBytes(), "v2".getBytes(), true); - transactionalKvBackend.put("testB".getBytes(), "v2".getBytes(), true); - transactionalKvBackend.commit(); - transactionalKvBackend.closeTransaction(); - - transactionalKvBackend.begin(); - transactionalKvBackend.put("testA".getBytes(), "v3".getBytes(), true); - transactionalKvBackend.delete("testC".getBytes()); - transactionalKvBackend.commit(); - transactionalKvBackend.closeTransaction(); - - // Test data is OK - transactionalKvBackend.begin(); - Assertions.assertEquals("v3", new String(transactionalKvBackend.get("testA".getBytes()))); - Assertions.assertEquals("v2", new String(transactionalKvBackend.get("testB".getBytes()))); - Assertions.assertNull(transactionalKvBackend.get("testC".getBytes())); - List> allData = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start("_".getBytes()) - .end("z".getBytes()) - .startInclusive(false) - .endInclusive(false) - .build()); - - // 7 for real-data(3 testA, 2 testB, 2 testC), 3 commit marks can't be seen as they start with - // 0x1E, last_timestamp can be seen as they have not been stored to the backend. - Assertions.assertEquals(7, allData.size()); - - // Set the TTL to 2 seconds before the kvGarbageCollector is created - Mockito.doReturn(2000L).when(config).get(STORE_DELETE_AFTER_TIME); - KvGarbageCollector kvGarbageCollector = new KvGarbageCollector(kvBackend, config, null); - - // Wait TTL time to make sure the data is expired, please see ENTITY_KV_TTL - Thread.sleep(3000); - kvGarbageCollector.collectAndClean(); - - allData = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start("_".getBytes()) - .end("z".getBytes()) - .startInclusive(false) - .endInclusive(false) - .build()); - // Except version 3 of testA and version 2 of testB will be left, all will be removed, so the - // left key-value paris will be 2(real-data) - Assertions.assertEquals(2, allData.size()); - Assertions.assertEquals("v3", new String(transactionalKvBackend.get("testA".getBytes()))); - Assertions.assertEquals("v2", new String(transactionalKvBackend.get("testB".getBytes()))); - Assertions.assertNull(transactionalKvBackend.get("testC".getBytes())); - } - } - - @Test - void testRemoveWithGCCollector1() throws IOException, InterruptedException { - Config config = getConfig(); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); - - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - if (!(store instanceof KvEntityStore)) { - return; - } - KvEntityStore kvEntityStore = (KvEntityStore) store; - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - - BaseMetalake metalake1 = createBaseMakeLake(1L, "metalake1", auditInfo); - CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake1"), "catalog1", auditInfo); - SchemaEntity schemaEntity = - createSchemaEntity(1L, Namespace.of("metalake1", "catalog1"), "schema1", auditInfo); - TableEntity tableEntity = - createTableEntity( - 1L, Namespace.of("metalake1", "catalog1", "schema1"), "table1", auditInfo); - FilesetEntity filesetEntity = - createFilesetEntity( - 1L, Namespace.of("metalake1", "catalog1", "schema1"), "fileset1", auditInfo); - - kvEntityStore.put(metalake1); - kvEntityStore.put(catalog); - kvEntityStore.put(schemaEntity); - kvEntityStore.put(tableEntity); - kvEntityStore.put(filesetEntity); - kvEntityStore.put( - UserEntity.builder() - .withId(1L) - .withAuditInfo(auditInfo) - .withName("the same") - .withNamespace(Namespace.of("metalake1", "catalog1", "schema1")) - .build()); - kvEntityStore.put( - GroupEntity.builder() - .withId(2L) - .withAuditInfo(auditInfo) - .withName("the same") - .withNamespace(Namespace.of("metalake1", "catalog1", "schema1")) - .build()); - - // now try to scan raw data from kv store - KvBackend kvBackend = kvEntityStore.backend; - List> data = - kvBackend.scan( - new KvRange.KvRangeBuilder() - .start("_".getBytes(StandardCharsets.UTF_8)) - .end("z".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build()); - - Assertions.assertEquals(7, data.size()); - - KvGarbageCollector kvGarbageCollector = kvEntityStore.kvGarbageCollector; - for (Pair pair : data) { - byte[] key = pair.getKey(); - LogHelper helper = kvGarbageCollector.decodeKey(key); - Assertions.assertNotSame(LogHelper.NONE, helper); - - switch (helper.type) { - case METALAKE: - Assertions.assertEquals(NameIdentifier.of("metalake1"), helper.identifier); - break; - case CATALOG: - Assertions.assertEquals(NameIdentifier.of("metalake1", "catalog1"), helper.identifier); - break; - case SCHEMA: - Assertions.assertEquals( - NameIdentifier.of("metalake1", "catalog1", "schema1"), helper.identifier); - break; - case TABLE: - Assertions.assertEquals( - NameIdentifier.of("metalake1", "catalog1", "schema1", "table1"), helper.identifier); - break; - case FILESET: - Assertions.assertEquals( - NameIdentifier.of("metalake1", "catalog1", "schema1", "fileset1"), - helper.identifier); - break; - case USER: - Assertions.assertEquals( - NameIdentifier.of("metalake1", "catalog1", "schema1", "the same"), - helper.identifier); - break; - - case GROUP: - Assertions.assertEquals( - NameIdentifier.of("metalake1", "catalog1", "schema1", "the same"), - helper.identifier); - break; - default: - Assertions.fail(); - } - } - } - } - - @Test - void testRemoveWithGCCollector2() throws IOException, InterruptedException { - Config config = getConfig(); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); - - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - - if (!(store instanceof KvEntityStore)) { - return; - } - KvEntityStore kvEntityStore = (KvEntityStore) store; - - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - - BaseMetalake metalake1 = createBaseMakeLake(1L, "metalake1", auditInfo); - BaseMetalake metalake2 = createBaseMakeLake(2L, "metalake2", auditInfo); - BaseMetalake metalake3 = createBaseMakeLake(3L, "metalake3", auditInfo); - - store.put(metalake1); - store.put(metalake2); - store.put(metalake3); - - store.delete(NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE); - - store.put(metalake1); - store.put(metalake2); - store.put(metalake3); - - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L); - Thread.sleep(1500); - - kvEntityStore.kvGarbageCollector.collectAndClean(); - - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE, BaseMetalake.class)); - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE, BaseMetalake.class)); - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE, BaseMetalake.class)); - - // Test catalog - CatalogEntity catalog1 = createCatalog(1L, Namespace.of("metalake1"), "catalog1", auditInfo); - CatalogEntity catalog2 = createCatalog(2L, Namespace.of("metalake1"), "catalog2", auditInfo); - - store.put(catalog1); - store.put(catalog2); - - store.delete(NameIdentifier.of("metalake1", "catalog1"), Entity.EntityType.CATALOG); - store.delete(NameIdentifier.of("metalake1", "catalog2"), Entity.EntityType.CATALOG); - - store.put(catalog1); - store.put(catalog2); - - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L); - Thread.sleep(1500); - - kvEntityStore.kvGarbageCollector.collectAndClean(); - - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake1", "catalog1"), - Entity.EntityType.CATALOG, - CatalogEntity.class)); - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake1", "catalog2"), - Entity.EntityType.CATALOG, - CatalogEntity.class)); - - // Test schema - SchemaEntity schema1 = - createSchemaEntity(1L, Namespace.of("metalake1", "catalog2"), "schema1", auditInfo); - SchemaEntity schema2 = - createSchemaEntity(2L, Namespace.of("metalake1", "catalog2"), "schema2", auditInfo); - - store.put(schema1); - store.put(schema2); - - store.delete(NameIdentifier.of("metalake1", "catalog2", "schema1"), Entity.EntityType.SCHEMA); - store.delete(NameIdentifier.of("metalake1", "catalog2", "schema2"), Entity.EntityType.SCHEMA); - - store.put(schema1); - store.put(schema2); - - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L); - Thread.sleep(1500); - kvEntityStore.kvGarbageCollector.collectAndClean(); - - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake1", "catalog2", "schema1"), - Entity.EntityType.SCHEMA, - SchemaEntity.class)); - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake1", "catalog2", "schema2"), - Entity.EntityType.SCHEMA, - SchemaEntity.class)); - - // Test table - TableEntity table1 = - createTableEntity( - 1L, Namespace.of("metalake1", "catalog2", "schema2"), "table1", auditInfo); - TableEntity table2 = - createTableEntity( - 2L, Namespace.of("metalake1", "catalog2", "schema2"), "table2", auditInfo); - - store.put(table1); - store.put(table2); - - store.delete( - NameIdentifier.of("metalake1", "catalog2", "schema2", "table1"), Entity.EntityType.TABLE); - store.delete( - NameIdentifier.of("metalake1", "catalog2", "schema2", "table2"), Entity.EntityType.TABLE); - - store.put(table1); - store.put(table2); - - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L); - Thread.sleep(1500); - kvEntityStore.kvGarbageCollector.collectAndClean(); - - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake1", "catalog2", "schema2", "table1"), - Entity.EntityType.TABLE, - TableEntity.class)); - Assertions.assertDoesNotThrow( - () -> - store.get( - NameIdentifier.of("metalake1", "catalog2", "schema2", "table2"), - Entity.EntityType.TABLE, - TableEntity.class)); - } - } - - @Test - void testIncrementalGC() throws Exception { - Config config = getConfig(); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); - - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - - if (!(store instanceof KvEntityStore)) { - return; - } - KvEntityStore kvEntityStore = (KvEntityStore) store; - - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - - BaseMetalake metalake1 = createBaseMakeLake(1L, "metalake1", auditInfo); - BaseMetalake metalake2 = createBaseMakeLake(2L, "metalake2", auditInfo); - BaseMetalake metalake3 = createBaseMakeLake(3L, "metalake3", auditInfo); - - for (int i = 0; i < 10; i++) { - store.put(metalake1); - store.put(metalake2); - store.put(metalake3); - - store.delete(NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE); - - Thread.sleep(10); - } - - store.put(metalake1); - store.put(metalake2); - store.put(metalake3); - - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(1000L); - Thread.sleep(1500); - - // Scan raw key-value data from storage to confirm the data is deleted - kvEntityStore.kvGarbageCollector.collectAndClean(); - List> allData = - kvEntityStore.backend.scan( - new KvRange.KvRangeBuilder() - .start("_".getBytes()) - .end("z".getBytes()) - .startInclusive(false) - .endInclusive(false) - .build()); - - Assertions.assertEquals(3, allData.size()); - - long transactionId = - getTransactionId( - getBinaryTransactionId(kvEntityStore.kvGarbageCollector.commitIdHasBeenCollected)); - Assertions.assertNotEquals(1, transactionId); - - for (int i = 0; i < 10; i++) { - store.delete(NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE); - store.put(metalake1); - store.put(metalake2); - store.put(metalake3); - Thread.sleep(10); - } - store.delete(NameIdentifier.of("metalake1"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake2"), Entity.EntityType.METALAKE); - store.delete(NameIdentifier.of("metalake3"), Entity.EntityType.METALAKE); - - Thread.sleep(1500); - kvEntityStore.kvGarbageCollector.collectAndClean(); - - allData = - kvEntityStore.backend.scan( - new KvRange.KvRangeBuilder() - .start("_".getBytes()) - .end("z".getBytes()) - .startInclusive(false) - .endInclusive(false) - .build()); - - Assertions.assertTrue(allData.isEmpty()); - - long transactionIdV2 = - getTransactionId( - getBinaryTransactionId(kvEntityStore.kvGarbageCollector.commitIdHasBeenCollected)); - Assertions.assertTrue(transactionIdV2 > transactionId); - } - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestKvNameMappingService.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestKvNameMappingService.java deleted file mode 100644 index 6736e8b8879..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestKvNameMappingService.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_STORE; -import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; -import static org.apache.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.nio.file.Files; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.EntitySerDeFactory; -import org.apache.gravitino.EntityStoreFactory; -import org.apache.gravitino.storage.IdGenerator; -import org.apache.gravitino.storage.NameMappingService; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -public class TestKvNameMappingService { - private Config getConfig() throws IOException { - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(3000L); - Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); - Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); - return config; - } - - private KvEntityStore getKvEntityStore(Config config) { - KvEntityStore kvEntityStore = (KvEntityStore) EntityStoreFactory.createEntityStore(config); - kvEntityStore.initialize(config); - kvEntityStore.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - return kvEntityStore; - } - - private IdGenerator getIdGeneratorByReflection(NameMappingService nameMappingService) - throws Exception { - Field field = nameMappingService.getClass().getDeclaredField("idGenerator"); - field.setAccessible(true); - IdGenerator idGenerator = (IdGenerator) field.get(nameMappingService); - IdGenerator spyIdGenerator = Mockito.spy(idGenerator); - field.set(nameMappingService, spyIdGenerator); - return spyIdGenerator; - } - - @Test - public void testGetIdByName() throws Exception { - try (KvEntityStore kvEntityStore = getKvEntityStore(getConfig())) { - NameMappingService nameMappingService = kvEntityStore.nameMappingService; - Assertions.assertNull(nameMappingService.getIdByName("name1")); - - IdGenerator spyIdGenerator = getIdGeneratorByReflection(nameMappingService); - Mockito.doReturn(1L).when(spyIdGenerator).nextId(); - - long name1Id = nameMappingService.getOrCreateIdFromName("name1"); - Long name1IdRead = nameMappingService.getIdByName("name1"); - Assertions.assertEquals(name1Id, name1IdRead); - - Assertions.assertNull(nameMappingService.getIdByName("name2")); - Mockito.doReturn(2L).when(spyIdGenerator).nextId(); - long name2Id = nameMappingService.getOrCreateIdFromName("name2"); - Long name2IdRead = nameMappingService.getIdByName("name2"); - Assertions.assertEquals(name2Id, name2IdRead); - } - } - - @Test - public void testUpdateName() throws Exception { - try (KvEntityStore kvEntityStore = getKvEntityStore(getConfig())) { - NameMappingService nameMappingService = kvEntityStore.nameMappingService; - IdGenerator idGenerator = getIdGeneratorByReflection(nameMappingService); - Mockito.doReturn(1L).when(idGenerator).nextId(); - long name1IdRead = nameMappingService.getOrCreateIdFromName("name1"); - Assertions.assertNotNull(nameMappingService.getIdByName("name1")); - - Mockito.doReturn(2L).when(idGenerator).nextId(); - long name2IdRead = nameMappingService.getOrCreateIdFromName("name2"); - Assertions.assertNotNull(nameMappingService.getIdByName("name1")); - Assertions.assertNotEquals(name1IdRead, name2IdRead); - - boolean result = nameMappingService.updateName("name1", "name3"); - Assertions.assertTrue(result); - - Long name3Id = nameMappingService.getIdByName("name3"); - Assertions.assertEquals(name1IdRead, name3Id); - Assertions.assertNull(nameMappingService.getIdByName("name1")); - - Assertions.assertFalse(nameMappingService.updateName("name1", "name4")); - } - } - - @Test - void testUpdateNameWithExistingName() throws Exception { - try (KvEntityStore kvEntityStore = getKvEntityStore(getConfig())) { - NameMappingService nameMappingService = kvEntityStore.nameMappingService; - IdGenerator idGenerator = getIdGeneratorByReflection(nameMappingService); - Mockito.doReturn(1L).when(idGenerator).nextId(); - long name1IdRead = nameMappingService.getOrCreateIdFromName("name1"); - Assertions.assertNotNull(nameMappingService.getIdByName("name1")); - - Mockito.doReturn(2L).when(idGenerator).nextId(); - long name2IdRead = nameMappingService.getOrCreateIdFromName("name2"); - Assertions.assertNotNull(nameMappingService.getIdByName("name1")); - Assertions.assertNotEquals(name1IdRead, name2IdRead); - - // Update name1 to an existing name like name2. - boolean result = nameMappingService.updateName("name1", "name2"); - Assertions.assertTrue(result); - - Long name2Id = nameMappingService.getIdByName("name2"); - Assertions.assertEquals(1L, name2Id); - - Assertions.assertNull(nameMappingService.getIdByName("name1")); - } - } - - @Test - public void testBindAndUnBind() throws Exception { - try (KvEntityStore kvEntityStore = getKvEntityStore(getConfig())) { - KvNameMappingService nameMappingService = - (KvNameMappingService) kvEntityStore.nameMappingService; - IdGenerator idGenerator = getIdGeneratorByReflection(nameMappingService); - - Mockito.doReturn(1L).when(idGenerator).nextId(); - nameMappingService.getOrCreateIdFromName("name1"); - Assertions.assertNotNull(nameMappingService.getIdByName("name1")); - - boolean result = nameMappingService.unbindNameAndId("name1"); - Assertions.assertTrue(result); - Assertions.assertNull(nameMappingService.getIdByName("name1")); - - Mockito.doReturn(2L).when(idGenerator).nextId(); - nameMappingService.getOrCreateIdFromName("name2"); - - TransactionalKvBackend spyKvBackend = Mockito.spy(nameMappingService.transactionalKvBackend); - // All deletes && puts will be converted to put operations. - Mockito.doThrow(new ArithmeticException()) - .when(spyKvBackend) - .put(Mockito.any(), Mockito.any(), Mockito.anyBoolean()); - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(3000L); - final NameMappingService mock = - new KvNameMappingService(spyKvBackend, nameMappingService.lock); - - // Now we try to use update. It should fail. - Assertions.assertThrowsExactly( - ArithmeticException.class, () -> mock.updateName("name2", "name3")); - Mockito.doCallRealMethod() - .when(spyKvBackend) - .put(Mockito.any(), Mockito.any(), Mockito.anyBoolean()); - Assertions.assertNull(mock.getIdByName("name3")); - Assertions.assertNotNull(mock.getIdByName("name2")); - } - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestRocksDBKvBackend.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestRocksDBKvBackend.java deleted file mode 100644 index 123f153a805..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestRocksDBKvBackend.java +++ /dev/null @@ -1,461 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.List; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.EntityAlreadyExistsException; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; - -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -public class TestRocksDBKvBackend { - - private KvBackend getKvBackEnd() throws IOException { - Config config = Mockito.mock(Config.class); - - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - - KvBackend kvBackend = new RocksDBKvBackend(); - kvBackend.initialize(config); - return kvBackend; - } - - @Test - void testStoragePath() { - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn("/a/b"); - RocksDBKvBackend kvBackend = new RocksDBKvBackend(); - String path = kvBackend.getStoragePath(config); - Assertions.assertEquals("/a/b", path); - - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(""); - kvBackend = new RocksDBKvBackend(); - path = kvBackend.getStoragePath(config); - // We haven't set the GRAVITINO_HOME - - String gravitinoHome = System.getenv("GRAVITINO_HOME"); - Assertions.assertEquals(gravitinoHome + "/data/rocksdb", path); - } - - @Test - void testPutAndGet() throws IOException, RocksDBException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "testKey".getBytes(StandardCharsets.UTF_8), - "testValue".getBytes(StandardCharsets.UTF_8), - false); - byte[] bytes = kvBackend.get("testKey".getBytes(StandardCharsets.UTF_8)); - - Assertions.assertNotNull(bytes); - Assertions.assertEquals("testValue", new String(bytes, StandardCharsets.UTF_8)); - - Assertions.assertThrowsExactly( - EntityAlreadyExistsException.class, - () -> - kvBackend.put( - "testKey".getBytes(StandardCharsets.UTF_8), - "testValue2".getBytes(StandardCharsets.UTF_8), - false)); - kvBackend.put( - "testKey".getBytes(StandardCharsets.UTF_8), - "testValue2".getBytes(StandardCharsets.UTF_8), - true); - - RocksDBKvBackend rocksDBKvBackend = (RocksDBKvBackend) kvBackend; - RocksDBKvBackend spyRocksDBKvBackend = Mockito.spy(rocksDBKvBackend); - Mockito.doThrow(new RocksDBException("Mock: Store file not found")) - .when(spyRocksDBKvBackend) - .handlePut(Mockito.any(), Mockito.any(), Mockito.anyBoolean()); - - Exception exception = - Assertions.assertThrowsExactly( - IOException.class, - () -> - spyRocksDBKvBackend.put( - "testKey".getBytes(StandardCharsets.UTF_8), - "testValue2".getBytes(StandardCharsets.UTF_8), - true)); - Assertions.assertTrue(exception.getMessage().contains("Mock: Store file not found")); - } - - @Test - void testDelete() throws IOException, RocksDBException { - KvBackend kvBackend = getKvBackEnd(); - Assertions.assertDoesNotThrow( - () -> kvBackend.delete("testKey".getBytes(StandardCharsets.UTF_8))); - kvBackend.put( - "testKey".getBytes(StandardCharsets.UTF_8), - "testValue".getBytes(StandardCharsets.UTF_8), - false); - Assertions.assertDoesNotThrow( - () -> kvBackend.delete("testKey".getBytes(StandardCharsets.UTF_8))); - Assertions.assertDoesNotThrow( - () -> kvBackend.delete("testKey".getBytes(StandardCharsets.UTF_8))); - - RocksDBKvBackend rocksDBKvBackend = (RocksDBKvBackend) kvBackend; - - RocksDB db = rocksDBKvBackend.getDb(); - RocksDB spyDb = Mockito.spy(db); - Mockito.doThrow(new RocksDBException("Mock: Network is unstable")) - .when(spyDb) - .delete(Mockito.any()); - rocksDBKvBackend.setDb(spyDb); - - Exception e = - Assertions.assertThrowsExactly( - IOException.class, () -> kvBackend.delete("testKey".getBytes(StandardCharsets.UTF_8))); - Assertions.assertTrue(e.getMessage().contains("Mock: Network is unstable")); - } - - @Test - void testDeleteRange() throws IOException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "abc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abd".getBytes(StandardCharsets.UTF_8), "abd".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abffff".getBytes(StandardCharsets.UTF_8), - "abffff".getBytes(StandardCharsets.UTF_8), - false); - kvBackend.put( - "abeee".getBytes(StandardCharsets.UTF_8), "abeee".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acc".getBytes(StandardCharsets.UTF_8), "acc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acca".getBytes(StandardCharsets.UTF_8), "acca".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accb".getBytes(StandardCharsets.UTF_8), "accb".getBytes(StandardCharsets.UTF_8), false); - - // More test case please refer to TestTransactionalKvBackend - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start("ab".getBytes(StandardCharsets.UTF_8)) - .end("ac".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build(); - - Assertions.assertDoesNotThrow(() -> kvBackend.deleteRange(kvRange)); - Assertions.assertNull(kvBackend.get("abc".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(kvBackend.get("abd".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(kvBackend.get("abffff".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(kvBackend.get("abeee".getBytes(StandardCharsets.UTF_8))); - - Assertions.assertNotNull(kvBackend.get("acc".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNotNull(kvBackend.get("acca".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNotNull(kvBackend.get("accb".getBytes(StandardCharsets.UTF_8))); - } - - @Test - void testDeleteRangeWhenIsStartExclusiveTrue() throws IOException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "abc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abd".getBytes(StandardCharsets.UTF_8), "abd".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abffff".getBytes(StandardCharsets.UTF_8), - "abffff".getBytes(StandardCharsets.UTF_8), - false); - kvBackend.put( - "abeee".getBytes(StandardCharsets.UTF_8), "abeee".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acc".getBytes(StandardCharsets.UTF_8), "acc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acca".getBytes(StandardCharsets.UTF_8), "acca".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accb".getBytes(StandardCharsets.UTF_8), "accb".getBytes(StandardCharsets.UTF_8), false); - - // More test case please refer to TestTransactionalKvBackend - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start("abc".getBytes(StandardCharsets.UTF_8)) - .end("abc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build(); - - Assertions.assertDoesNotThrow(() -> kvBackend.deleteRange(kvRange)); - Assertions.assertNull(kvBackend.get("abc".getBytes(StandardCharsets.UTF_8))); - - Assertions.assertNotNull(kvBackend.get("acc".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNotNull(kvBackend.get("acca".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNotNull(kvBackend.get("accb".getBytes(StandardCharsets.UTF_8))); - } - - @Test - void testDeleteRangeWhenIsEndExclusiveTrue() throws IOException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "abc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abd".getBytes(StandardCharsets.UTF_8), "abd".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abffff".getBytes(StandardCharsets.UTF_8), - "abffff".getBytes(StandardCharsets.UTF_8), - false); - kvBackend.put( - "abeee".getBytes(StandardCharsets.UTF_8), "abeee".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acc".getBytes(StandardCharsets.UTF_8), "acc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acca".getBytes(StandardCharsets.UTF_8), "acca".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accb".getBytes(StandardCharsets.UTF_8), "accb".getBytes(StandardCharsets.UTF_8), false); - - // More test case please refer to TestTransactionalKvBackend - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start("ab".getBytes(StandardCharsets.UTF_8)) - .end("abc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(true) - .build(); - - Assertions.assertDoesNotThrow(() -> kvBackend.deleteRange(kvRange)); - - Assertions.assertNotNull(kvBackend.get("acc".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNotNull(kvBackend.get("acca".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNotNull(kvBackend.get("accb".getBytes(StandardCharsets.UTF_8))); - } - - @Test - void testScanWithBrokenRocksDB() throws IOException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "abc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abd".getBytes(StandardCharsets.UTF_8), "abd".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abffff".getBytes(StandardCharsets.UTF_8), - "abffff".getBytes(StandardCharsets.UTF_8), - false); - kvBackend.put( - "abeee".getBytes(StandardCharsets.UTF_8), "abeee".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acc".getBytes(StandardCharsets.UTF_8), "acc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acca".getBytes(StandardCharsets.UTF_8), "acca".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accb".getBytes(StandardCharsets.UTF_8), "accb".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accg".getBytes(StandardCharsets.UTF_8), "accg".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acf".getBytes(StandardCharsets.UTF_8), "acf".getBytes(StandardCharsets.UTF_8), false); - - // More test case please refer to TestTransactionalKvBackend - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start("ab".getBytes(StandardCharsets.UTF_8)) - .end("ac".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build(); - - List> data = kvBackend.scan(kvRange); - Assertions.assertEquals(4, data.size()); - - RocksDBKvBackend rocksDBKvBackend = (RocksDBKvBackend) kvBackend; - RocksDB db = rocksDBKvBackend.getDb(); - RocksDB spyDb = Mockito.spy(db); - - Mockito.when(spyDb.newIterator()).thenThrow(new RuntimeException("Mock: RocksDB is broken")); - rocksDBKvBackend.setDb(spyDb); - - Exception e = - Assertions.assertThrowsExactly(RuntimeException.class, () -> kvBackend.scan(kvRange)); - Assertions.assertTrue(e.getMessage().contains("Mock: RocksDB is broken")); - } - - @Test - void testScanWithBrokenRocksDWhenIsStartExclusiveTrue() throws IOException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "abc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abd".getBytes(StandardCharsets.UTF_8), "abd".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abffff".getBytes(StandardCharsets.UTF_8), - "abffff".getBytes(StandardCharsets.UTF_8), - false); - kvBackend.put( - "abeee".getBytes(StandardCharsets.UTF_8), "abeee".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acc".getBytes(StandardCharsets.UTF_8), "acc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acca".getBytes(StandardCharsets.UTF_8), "acca".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accb".getBytes(StandardCharsets.UTF_8), "accb".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accg".getBytes(StandardCharsets.UTF_8), "accg".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acf".getBytes(StandardCharsets.UTF_8), "acf".getBytes(StandardCharsets.UTF_8), false); - - // More test case please refer to TestTransactionalKvBackend - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start("abc".getBytes(StandardCharsets.UTF_8)) - .end("ac".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build(); - - List> data = kvBackend.scan(kvRange); - Assertions.assertEquals(4, data.size()); - - RocksDBKvBackend rocksDBKvBackend = (RocksDBKvBackend) kvBackend; - RocksDB db = rocksDBKvBackend.getDb(); - RocksDB spyDb = Mockito.spy(db); - - Mockito.when(spyDb.newIterator()).thenThrow(new RuntimeException("Mock: RocksDB is broken")); - rocksDBKvBackend.setDb(spyDb); - - Exception e = - Assertions.assertThrowsExactly(RuntimeException.class, () -> kvBackend.scan(kvRange)); - Assertions.assertTrue(e.getMessage().contains("Mock: RocksDB is broken")); - } - - @Test - void testScanWithBrokenRocksDBWhenIsEndExclusiveTrue() throws IOException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "abc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abd".getBytes(StandardCharsets.UTF_8), "abd".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abffff".getBytes(StandardCharsets.UTF_8), - "abffff".getBytes(StandardCharsets.UTF_8), - false); - kvBackend.put( - "abeee".getBytes(StandardCharsets.UTF_8), "abeee".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acc".getBytes(StandardCharsets.UTF_8), "acc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acca".getBytes(StandardCharsets.UTF_8), "acca".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accb".getBytes(StandardCharsets.UTF_8), "accb".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accg".getBytes(StandardCharsets.UTF_8), "accg".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acf".getBytes(StandardCharsets.UTF_8), "acf".getBytes(StandardCharsets.UTF_8), false); - - // More test case please refer to TestTransactionalKvBackend - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start("ab".getBytes(StandardCharsets.UTF_8)) - .end("abc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(true) - .build(); - - List> data = kvBackend.scan(kvRange); - Assertions.assertEquals(1, data.size()); - - RocksDBKvBackend rocksDBKvBackend = (RocksDBKvBackend) kvBackend; - RocksDB db = rocksDBKvBackend.getDb(); - RocksDB spyDb = Mockito.spy(db); - - Mockito.when(spyDb.newIterator()).thenThrow(new RuntimeException("Mock: RocksDB is broken")); - rocksDBKvBackend.setDb(spyDb); - - Exception e = - Assertions.assertThrowsExactly(RuntimeException.class, () -> kvBackend.scan(kvRange)); - Assertions.assertTrue(e.getMessage().contains("Mock: RocksDB is broken")); - } - - @Test - void testScan() throws IOException { - KvBackend kvBackend = getKvBackEnd(); - kvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "abc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abd".getBytes(StandardCharsets.UTF_8), "abd".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "abffff".getBytes(StandardCharsets.UTF_8), - "abffff".getBytes(StandardCharsets.UTF_8), - false); - kvBackend.put( - "abeee".getBytes(StandardCharsets.UTF_8), "abeee".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acc".getBytes(StandardCharsets.UTF_8), "acc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acca".getBytes(StandardCharsets.UTF_8), "acca".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accb".getBytes(StandardCharsets.UTF_8), "accb".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "accg".getBytes(StandardCharsets.UTF_8), "accg".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "acf".getBytes(StandardCharsets.UTF_8), "acf".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "adc".getBytes(StandardCharsets.UTF_8), "adc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "add".getBytes(StandardCharsets.UTF_8), "add".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "ae".getBytes(StandardCharsets.UTF_8), "ae".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "aef".getBytes(StandardCharsets.UTF_8), "aef".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "ag".getBytes(StandardCharsets.UTF_8), "ag".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "b".getBytes(StandardCharsets.UTF_8), "b".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "ba".getBytes(StandardCharsets.UTF_8), "ba".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "bc".getBytes(StandardCharsets.UTF_8), "bc".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "c".getBytes(StandardCharsets.UTF_8), "f".getBytes(StandardCharsets.UTF_8), false); - kvBackend.put( - "f".getBytes(StandardCharsets.UTF_8), "f".getBytes(StandardCharsets.UTF_8), false); - - KvRange kvRange = - new KvRange.KvRangeBuilder() - .start("ab".getBytes(StandardCharsets.UTF_8)) - .end("ac".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build(); - - List> data = kvBackend.scan(kvRange); - Assertions.assertEquals(4, data.size()); - - RocksDBKvBackend rocksDBKvBackend = (RocksDBKvBackend) kvBackend; - RocksDB db = rocksDBKvBackend.getDb(); - RocksDB spyDb = Mockito.spy(db); - - Mockito.when(spyDb.newIterator()).thenCallRealMethod(); - Assertions.assertDoesNotThrow(() -> kvBackend.scan(kvRange)); - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestStorageVersion.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestStorageVersion.java deleted file mode 100644 index 84e0713edeb..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestStorageVersion.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.ENTITY_KV_STORE; -import static org.apache.gravitino.Configs.ENTITY_STORE; -import static org.apache.gravitino.Configs.STORE_DELETE_AFTER_TIME; -import static org.apache.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.EntitySerDeFactory; -import org.apache.gravitino.EntityStore; -import org.apache.gravitino.EntityStoreFactory; -import org.apache.gravitino.storage.StorageLayoutException; -import org.apache.gravitino.storage.StorageLayoutVersion; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -class TestStorageVersion { - - @Test - void testFromString() { - StorageLayoutVersion version = StorageLayoutVersion.fromString("v1"); - Assertions.assertEquals(StorageLayoutVersion.V1, version); - - Assertions.assertThrowsExactly( - StorageLayoutException.class, () -> StorageLayoutVersion.fromString("v200000.0")); - } - - @Test - void testStorageLayoutVersion() throws IOException { - Config config = Mockito.mock(Config.class); - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); - Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); - - // First time create entity store, the storage layout version should be DEFAULT_LAYOUT_VERSION - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - Assertions.assertTrue(store instanceof KvEntityStore); - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - KvEntityStore entityStore = (KvEntityStore) store; - Assertions.assertEquals(StorageLayoutVersion.V1, entityStore.storageLayoutVersion); - } - - // Second time create entity store, the storage layout version should be DEFAULT_LAYOUT_VERSION - try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { - store.initialize(config); - Assertions.assertTrue(store instanceof KvEntityStore); - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config.get(Configs.ENTITY_SERDE))); - KvEntityStore entityStore = (KvEntityStore) store; - Assertions.assertEquals(StorageLayoutVersion.V1, entityStore.storageLayoutVersion); - } - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionIdGenerator.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionIdGenerator.java deleted file mode 100644 index 70330d48f0c..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionIdGenerator.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.storage.TransactionIdGenerator; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -public class TestTransactionIdGenerator { - - private static final Logger LOGGER = LoggerFactory.getLogger(TestTransactionalKvBackend.class); - - private Config getConfig() throws IOException { - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(3000L); - return config; - } - - private KvBackend getKvBackEnd(Config config) throws IOException { - KvBackend kvBackend = new RocksDBKvBackend(); - kvBackend.initialize(config); - return kvBackend; - } - - @Test - void testSchedulerAndSkewTime() throws IOException, InterruptedException { - Config config = getConfig(); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - transactionIdGenerator.start(); - // Make sure the scheduler has schedule once - Thread.sleep(2000 + 500); - Assertions.assertNotNull(kvBackend.get(TransactionIdGeneratorImpl.LAST_TIMESTAMP)); - } - - @Test - void testNextId() throws IOException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - transactionIdGenerator.start(); - - long id1 = transactionIdGenerator.nextId(); - long id2 = transactionIdGenerator.nextId(); - - // Test that nextId generates different IDs - Assertions.assertNotEquals(id1, id2); - // Test that nextId generates increasing IDs - Assertions.assertTrue(id2 > id1); - - // Test that incrementId reset to 0 after reaching its maximum value - for (int i = 2; i < (1 << 18); i++) { - transactionIdGenerator.nextId(); - } - long idAfterReset = transactionIdGenerator.nextId(); - Assertions.assertTrue(idAfterReset > id2); - - // Test that nextId generates increasing IDs even after incrementId reset - long idAfterReset2 = transactionIdGenerator.nextId(); - Assertions.assertTrue(idAfterReset2 > idAfterReset); - } - - @ParameterizedTest - @ValueSource(ints = {16}) - @Disabled("It's very time-consuming, so we disable it by default.") - void testTransactionIdGeneratorQPS(int threadNum) throws IOException, InterruptedException { - Config config = getConfig(); - String path = config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH); - LOGGER.info("testTransactionIdGeneratorQPS path: {}", path); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - ThreadPoolExecutor threadPoolExecutor = - new ThreadPoolExecutor( - threadNum, - threadNum, - 1, - TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1000), - new ThreadFactoryBuilder() - .setDaemon(false) - .setNameFormat("testTransactionIdGenerator-%d") - .build()); - - AtomicLong atomicLong = new AtomicLong(0); - for (int i = 0; i < threadNum; i++) { - threadPoolExecutor.execute( - () -> { - long current = System.currentTimeMillis(); - while (System.currentTimeMillis() - current <= 2000) { - transactionIdGenerator.nextId(); - atomicLong.getAndIncrement(); - } - }); - } - Thread.sleep(100); - threadPoolExecutor.shutdown(); - threadPoolExecutor.awaitTermination(5, TimeUnit.SECONDS); - LOGGER.info(String.format("%d thread qps is: %d/s", threadNum, atomicLong.get() / 2)); - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionalKvBackend.java b/core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionalKvBackend.java deleted file mode 100644 index 2528c78ec96..00000000000 --- a/core/src/test/java/org/apache/gravitino/storage/kv/TestTransactionalKvBackend.java +++ /dev/null @@ -1,848 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.storage.kv; - -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static org.apache.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; - -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.gravitino.Config; -import org.apache.gravitino.Configs; -import org.apache.gravitino.storage.TransactionIdGenerator; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings("DefaultCharset") -@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") -class TestTransactionalKvBackend { - - private static final Logger LOGGER = LoggerFactory.getLogger(TestTransactionalKvBackend.class); - - private Config getConfig() throws IOException { - File baseDir = new File(System.getProperty("java.io.tmpdir")); - File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); - file.deleteOnExit(); - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(file.getAbsolutePath()); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(3000L); - return config; - } - - private KvBackend getKvBackEnd(Config config) throws IOException { - KvBackend kvBackend = new RocksDBKvBackend(); - kvBackend.initialize(config); - return kvBackend; - } - - @Test - void testGet() throws IOException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackend transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - transactionalKvBackend.begin(); - transactionalKvBackend.put("key1".getBytes(), "value1".getBytes(), true); - transactionalKvBackend.put("key2".getBytes(), "value2".getBytes(), true); - transactionalKvBackend.commit(); - - transactionalKvBackend.begin(); - Assertions.assertEquals("value1", new String(transactionalKvBackend.get("key1".getBytes()))); - Assertions.assertEquals("value2", new String(transactionalKvBackend.get("key2".getBytes()))); - - transactionalKvBackend.begin(); - transactionalKvBackend.put("key1".getBytes(), "value3".getBytes(), true); - transactionalKvBackend.put("key2".getBytes(), "value4".getBytes(), true); - transactionalKvBackend.commit(); - transactionalKvBackend.close(); - - transactionalKvBackend.begin(); - Assertions.assertEquals("value3", new String(transactionalKvBackend.get("key1".getBytes()))); - Assertions.assertEquals("value4", new String(transactionalKvBackend.get("key2".getBytes()))); - transactionalKvBackend.close(); - transactionIdGenerator.close(); - } - - @Test - void testDelete() throws IOException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackend transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - transactionalKvBackend.begin(); - transactionalKvBackend.put("key1".getBytes(), "value1".getBytes(), true); - transactionalKvBackend.put("key2".getBytes(), "value2".getBytes(), true); - transactionalKvBackend.commit(); - - transactionalKvBackend.begin(); - transactionalKvBackend.delete("key1".getBytes()); - transactionalKvBackend.commit(); - - transactionalKvBackend.begin(); - Assertions.assertNull(transactionalKvBackend.get("key1".getBytes())); - Assertions.assertNotNull(transactionalKvBackend.get("key2".getBytes())); - - transactionalKvBackend.begin(); - transactionalKvBackend.delete("key2".getBytes()); - transactionalKvBackend.commit(); - - Assertions.assertNull(transactionalKvBackend.get("key1".getBytes())); - Assertions.assertNull(transactionalKvBackend.get("key2".getBytes())); - transactionalKvBackend.close(); - transactionIdGenerator.close(); - } - - @Test - void testScan() throws IOException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackend transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - transactionalKvBackend.begin(); - - transactionalKvBackend.put("key1".getBytes(), "value1".getBytes(), true); - transactionalKvBackend.put("key2".getBytes(), "value2".getBytes(), true); - transactionalKvBackend.put("key3".getBytes(), "value3".getBytes(), true); - transactionalKvBackend.commit(); - transactionalKvBackend.close(); - - Map map = - new HashMap() { - { - put("key1", "value1"); - put("key2", "value2"); - put("key3", "value3"); - } - }; - - transactionalKvBackend = - new TransactionalKvBackendImpl( - kvBackend, new TransactionIdGeneratorImpl(kvBackend, config)); - transactionalKvBackend.begin(); - List> pairs = - transactionalKvBackend.scan( - new KvRange.KvRangeBuilder() - .start("k".getBytes()) - .end("kez".getBytes()) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(3, pairs.size()); - for (Pair pair : pairs) { - Assertions.assertEquals(map.get(new String(pair.getKey())), new String(pair.getValue())); - } - List resultList = - pairs.stream().map(p -> new String(p.getKey())).collect(Collectors.toList()); - Assertions.assertEquals(Lists.newArrayList("key1", "key2", "key3"), resultList); - - pairs = - transactionalKvBackend.scan( - new KvRange.KvRangeBuilder() - .start("key1".getBytes()) - .end("kez".getBytes()) - .startInclusive(false) - .endInclusive(true) - .build()); - Assertions.assertEquals(2, pairs.size()); - for (Pair pair : pairs) { - Assertions.assertEquals(map.get(new String(pair.getKey())), new String(pair.getValue())); - } - resultList = pairs.stream().map(p -> new String(p.getKey())).collect(Collectors.toList()); - Assertions.assertEquals(Lists.newArrayList("key2", "key3"), resultList); - - pairs = - transactionalKvBackend.scan( - new KvRange.KvRangeBuilder() - .start("key1".getBytes()) - .end("key3".getBytes()) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(1, pairs.size()); - for (Pair pair : pairs) { - Assertions.assertEquals(map.get(new String(pair.getKey())), new String(pair.getValue())); - } - resultList = pairs.stream().map(p -> new String(p.getKey())).collect(Collectors.toList()); - Assertions.assertEquals(Lists.newArrayList("key2"), resultList); - - pairs = - transactionalKvBackend.scan( - new KvRange.KvRangeBuilder() - .start("key3".getBytes()) - .end("kez".getBytes()) - .startInclusive(true) - .endInclusive(true) - .build()); - Assertions.assertEquals(1, pairs.size()); - for (Pair pair : pairs) { - Assertions.assertEquals(map.get(new String(pair.getKey())), new String(pair.getValue())); - } - resultList = pairs.stream().map(p -> new String(p.getKey())).collect(Collectors.toList()); - Assertions.assertEquals(Lists.newArrayList("key3"), resultList); - - pairs = - transactionalKvBackend.scan( - new KvRange.KvRangeBuilder() - .start("kf".getBytes()) - .end("kg".getBytes()) - .startInclusive(true) - .endInclusive(true) - .build()); - Assertions.assertEquals(0, pairs.size()); - transactionIdGenerator.close(); - } - - @Test - void testDeleteRange() throws IOException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackend transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - transactionalKvBackend.begin(); - transactionalKvBackend.put("key1".getBytes(), "value1".getBytes(), true); - transactionalKvBackend.put("key2".getBytes(), "value2".getBytes(), true); - transactionalKvBackend.put("key3".getBytes(), "value3".getBytes(), true); - transactionalKvBackend.commit(); - - transactionalKvBackend.begin(); - transactionalKvBackend.deleteRange( - new KvRange.KvRangeBuilder() - .start("key1".getBytes()) - .end("key3".getBytes()) - .startInclusive(true) - .endInclusive(true) - .build()); - transactionalKvBackend.commit(); - - transactionalKvBackend.begin(); - Assertions.assertNull(transactionalKvBackend.get("key1".getBytes())); - Assertions.assertNull(transactionalKvBackend.get("key2".getBytes())); - Assertions.assertNull(transactionalKvBackend.get("key3".getBytes())); - transactionalKvBackend.close(); - transactionIdGenerator.close(); - } - - @Test - void testException() throws IOException, InterruptedException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackendImpl kvTransactionManager = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - kvTransactionManager.begin(); - List> pairs = - Lists.newArrayList( - Pair.of( - kvTransactionManager.constructKey("key1".getBytes()), - kvTransactionManager.constructValue("value1".getBytes(), ValueStatusEnum.NORMAL)), - Pair.of( - kvTransactionManager.constructKey("key2".getBytes()), - kvTransactionManager.constructValue("value2".getBytes(), ValueStatusEnum.NORMAL)), - Pair.of( - kvTransactionManager.constructKey("key3".getBytes()), - kvTransactionManager.constructValue("value3".getBytes(), ValueStatusEnum.NORMAL)), - Pair.of( - kvTransactionManager.constructKey("key4".getBytes()), - kvTransactionManager.constructValue("value4".getBytes(), ValueStatusEnum.NORMAL)), - Pair.of( - kvTransactionManager.constructKey("key5".getBytes()), - kvTransactionManager.constructValue("value6".getBytes(), ValueStatusEnum.NORMAL)), - Pair.of( - kvTransactionManager.constructKey("key6".getBytes()), - kvTransactionManager.constructValue("value7".getBytes(), ValueStatusEnum.NORMAL)), - Pair.of( - kvTransactionManager.constructKey("key7".getBytes()), - kvTransactionManager.constructValue("value8".getBytes(), ValueStatusEnum.NORMAL)), - Pair.of( - kvTransactionManager.constructKey("key8".getBytes()), - kvTransactionManager.constructValue("value9".getBytes(), ValueStatusEnum.NORMAL)), - - // Will throw NPE to roll back the transaction. - Pair.of(kvTransactionManager.constructKey("key9".getBytes()), null)); - - Pair[] arrayPair = pairs.toArray(new Pair[0]); - - kvTransactionManager = new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - for (int i = 0; i < 10000; i++) { - ArrayUtils.shuffle(arrayPair); - - kvTransactionManager - .putPairs - .get() - .addAll(Arrays.stream(arrayPair).collect(Collectors.toList())); - Assertions.assertThrows(Exception.class, kvTransactionManager::commit); - - kvTransactionManager.begin(); - Assertions.assertNull(kvTransactionManager.get("key1".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key2".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key3".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key4".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key5".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key6".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key7".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key8".getBytes())); - Assertions.assertNull(kvTransactionManager.get("key9".getBytes())); - } - Thread.sleep(1000); - transactionIdGenerator.close(); - } - - @ParameterizedTest - @ValueSource(ints = {16}) - @Disabled("It's very time-consuming, so we disable it by default.") - void testConcurrentRead(int threadNum) throws IOException, InterruptedException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackend transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - transactionalKvBackend.begin(); - for (int i = 0; i < threadNum; i++) { - transactionalKvBackend.put( - ("key" + i).getBytes(StandardCharsets.UTF_8), - ("value" + i).getBytes(StandardCharsets.UTF_8), - true); - } - transactionalKvBackend.commit(); - transactionalKvBackend.closeTransaction(); - - ThreadPoolExecutor threadPoolExecutor = - new ThreadPoolExecutor( - threadNum, - threadNum, - 1, - TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1000), - new ThreadFactoryBuilder() - .setDaemon(false) - .setNameFormat("testTransactionIdGenerator-%d") - .build()); - - ThreadLocalRandom random = ThreadLocalRandom.current(); - long current = System.currentTimeMillis(); - AtomicLong atomicLong = new AtomicLong(0); - threadPoolExecutor.submit( - () -> { - while (System.currentTimeMillis() - current <= 2000) { - transactionalKvBackend.begin(); - int i = random.nextInt(threadNum) + 1; - byte[] binaryValue = - Assertions.assertDoesNotThrow( - () -> transactionalKvBackend.get(("key" + i).getBytes(StandardCharsets.UTF_8))); - Assertions.assertEquals(("value" + i).getBytes(StandardCharsets.UTF_8), binaryValue); - transactionalKvBackend.closeTransaction(); - atomicLong.getAndIncrement(); - } - }); - - Thread.sleep(100); - threadPoolExecutor.shutdown(); - threadPoolExecutor.awaitTermination(5, TimeUnit.SECONDS); - LOGGER.info(String.format("%d thread qps is: %d/s", threadNum, atomicLong.get() / 2)); - } - - @ParameterizedTest - @ValueSource(ints = {1, 2, 4, 16, 32}) - @Disabled("It's very time-consuming, so we disable it by default.") - void testConcurrentWrite(int threadNum) throws IOException, InterruptedException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackend transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - ThreadPoolExecutor threadPoolExecutor = - new ThreadPoolExecutor( - threadNum, - threadNum, - 1, - TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1000), - new ThreadFactoryBuilder() - .setDaemon(false) - .setNameFormat("testTransactionIdGenerator-%d") - .build()); - - ThreadLocalRandom random = ThreadLocalRandom.current(); - long current = System.currentTimeMillis(); - AtomicLong atomicLong = new AtomicLong(0); - threadPoolExecutor.submit( - () -> { - while (System.currentTimeMillis() - current <= 2000) { - transactionalKvBackend.begin(); - int i = random.nextInt(threadNum) + 1; - Assertions.assertDoesNotThrow( - () -> - transactionalKvBackend.put( - ("key" + i).getBytes(StandardCharsets.UTF_8), - ("value" + i).getBytes(StandardCharsets.UTF_8), - true)); - transactionalKvBackend.closeTransaction(); - atomicLong.getAndIncrement(); - } - }); - - Thread.sleep(100); - threadPoolExecutor.shutdown(); - threadPoolExecutor.awaitTermination(5, TimeUnit.SECONDS); - LOGGER.info(String.format("%d thread write qps is: %d/s", threadNum, atomicLong.get() / 2)); - } - - @Test - void testPrefixError() throws IOException, InterruptedException { - Config config = getConfig(); - KvBackend kvBackend = getKvBackEnd(config); - TransactionIdGenerator transactionIdGenerator = - new TransactionIdGeneratorImpl(kvBackend, config); - TransactionalKvBackend transactionalKvBackend = - new TransactionalKvBackendImpl(kvBackend, transactionIdGenerator); - - transactionalKvBackend.begin(); - transactionalKvBackend.put( - "abcefghi ".getBytes(StandardCharsets.UTF_8), - "value1".getBytes(StandardCharsets.UTF_8), - true); - transactionalKvBackend.commit(); - - // Start to read; - transactionalKvBackend.begin(); - Assertions.assertEquals( - "value1", new String(transactionalKvBackend.get("abcefghi ".getBytes()))); - Assertions.assertNull(transactionalKvBackend.get("abcefghi".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abcefgh".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abcefg".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abcef".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abce".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abc".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("ab".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("a".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("_".getBytes(StandardCharsets.UTF_8))); - - transactionalKvBackend.begin(); - transactionalKvBackend.put( - "abc".getBytes(StandardCharsets.UTF_8), "value1".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.commit(); - - transactionalKvBackend.begin(); - Assertions.assertEquals("value1", new String(transactionalKvBackend.get("abc".getBytes()))); - Assertions.assertNull(transactionalKvBackend.get("ab".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("a".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abce".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abcd".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abcef".getBytes(StandardCharsets.UTF_8))); - Assertions.assertNull(transactionalKvBackend.get("abcefg".getBytes(StandardCharsets.UTF_8))); - transactionalKvBackend.commit(); - - // Start to test scan - // We randomly repeated insert some data and test the result - int repeatedTimes = new Random().nextInt(10) + 1; - for (int i = 0; i < repeatedTimes; i++) { - transactionalKvBackend.begin(); - transactionalKvBackend.put( - "m".getBytes(StandardCharsets.UTF_8), "value1".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "mb".getBytes(StandardCharsets.UTF_8), "value2".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "mbc".getBytes(StandardCharsets.UTF_8), "value3".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "mbcd".getBytes(StandardCharsets.UTF_8), "value4".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "mbcde".getBytes(StandardCharsets.UTF_8), - "value5".getBytes(StandardCharsets.UTF_8), - true); - transactionalKvBackend.put( - "mbcdf".getBytes(StandardCharsets.UTF_8), - "value6".getBytes(StandardCharsets.UTF_8), - true); - transactionalKvBackend.put( - "mbcdfg".getBytes(StandardCharsets.UTF_8), - "value7".getBytes(StandardCharsets.UTF_8), - true); - transactionalKvBackend.commit(); - Thread.sleep(1); - } - - transactionalKvBackend.begin(); - - // start is 'mb' and end is 'mc' - List> data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build()); - Assertions.assertEquals(6, data.size()); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(5, data.size()); - Assertions.assertEquals("mbc", new String(data.get(0).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - Assertions.assertEquals(6, data.size()); - Assertions.assertEquals("mb", new String(data.get(0).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build()); - Assertions.assertEquals(6, data.size()); - Assertions.assertEquals("mb", new String(data.get(0).getKey())); - - // Start is 'mb' and end is 'mbcde' - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbcde".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(2, data.size()); - Assertions.assertEquals("mbcd", new String(data.get(1).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbcde".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(true) - .build()); - Assertions.assertEquals(3, data.size()); - Assertions.assertEquals("mbcde", new String(data.get(2).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbcde".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - - Assertions.assertEquals(4, data.size()); - Assertions.assertEquals("mbcde", new String(data.get(3).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbcde".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build()); - - Assertions.assertEquals(3, data.size()); - Assertions.assertEquals("mbcd", new String(data.get(2).getKey())); - - // Start is 'mb' and end is 'mbc' - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(true) - .build()); - Assertions.assertEquals(1, data.size()); - Assertions.assertEquals("mbc", new String(data.get(0).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(0, data.size()); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build()); - Assertions.assertEquals(1, data.size()); - Assertions.assertEquals("mb", new String(data.get(0).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mbc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - Assertions.assertEquals(2, data.size()); - Assertions.assertEquals("mb", new String(data.get(0).getKey())); - Assertions.assertEquals("mbc", new String(data.get(1).getKey())); - - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mb".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - - Assertions.assertEquals(1, data.size()); - Assertions.assertEquals("mb", new String(data.get(0).getKey())); - - transactionalKvBackend.scan( - KvRange.builder() - .start("mb".getBytes(StandardCharsets.UTF_8)) - .end("mc".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - - Assertions.assertEquals(1, data.size()); - Assertions.assertEquals("mb", new String(data.get(0).getKey())); - - repeatedTimes = new Random().nextInt(10) + 1; - for (int i = 0; i < repeatedTimes; i++) { - transactionalKvBackend.begin(); - transactionalKvBackend.put( - "bc".getBytes(StandardCharsets.UTF_8), "value1".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "bcd".getBytes(StandardCharsets.UTF_8), "value2".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "bcde".getBytes(StandardCharsets.UTF_8), "value3".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "bce".getBytes(StandardCharsets.UTF_8), "value4".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "bcef".getBytes(StandardCharsets.UTF_8), "value5".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "bef".getBytes(StandardCharsets.UTF_8), "value6".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.put( - "beg".getBytes(StandardCharsets.UTF_8), "value7".getBytes(StandardCharsets.UTF_8), true); - transactionalKvBackend.commit(); - Thread.sleep(1); - } - - transactionalKvBackend.begin(); - - // Start is "bc" and end is "bcef" - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bcef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(3, data.size()); - Assertions.assertEquals("bcd", new String(data.get(0).getKey())); - Assertions.assertEquals("bce", new String(data.get(2).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bcef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(true) - .build()); - Assertions.assertEquals(4, data.size()); - Assertions.assertEquals("bcd", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(3).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bcef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - Assertions.assertEquals(5, data.size()); - Assertions.assertEquals("bc", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(4).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bcef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build()); - Assertions.assertEquals(4, data.size()); - Assertions.assertEquals("bc", new String(data.get(0).getKey())); - Assertions.assertEquals("bce", new String(data.get(3).getKey())); - - // Start is "bc" and end is "bef" - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(4, data.size()); - Assertions.assertEquals("bcd", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(3).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(true) - .build()); - Assertions.assertEquals(5, data.size()); - Assertions.assertEquals("bcd", new String(data.get(0).getKey())); - Assertions.assertEquals("bef", new String(data.get(4).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - Assertions.assertEquals(6, data.size()); - Assertions.assertEquals("bc", new String(data.get(0).getKey())); - Assertions.assertEquals("bef", new String(data.get(5).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build()); - Assertions.assertEquals(5, data.size()); - Assertions.assertEquals("bc", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(4).getKey())); - - // Start is "bc" and end is "be" - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("be".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(false) - .build()); - Assertions.assertEquals(4, data.size()); - Assertions.assertEquals("bcd", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(3).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("be".getBytes(StandardCharsets.UTF_8)) - .startInclusive(false) - .endInclusive(true) - .build()); - Assertions.assertEquals(4, data.size()); - Assertions.assertEquals("bcd", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(3).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("be".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(true) - .build()); - Assertions.assertEquals(5, data.size()); - Assertions.assertEquals("bc", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(4).getKey())); - data = - transactionalKvBackend.scan( - KvRange.builder() - .start("bc".getBytes(StandardCharsets.UTF_8)) - .end("bef".getBytes(StandardCharsets.UTF_8)) - .startInclusive(true) - .endInclusive(false) - .build()); - Assertions.assertEquals(5, data.size()); - Assertions.assertEquals("bc", new String(data.get(0).getKey())); - Assertions.assertEquals("bcef", new String(data.get(4).getKey())); - } - - @Test - void testRevertByte() { - byte[] bytes = {0x01, 0x02, 0x03, 0x04}; - byte[] copy = ArrayUtils.clone(bytes); - byte[] revertBytes = TransactionalKvBackendImpl.revertByteArray(bytes); - Assertions.assertArrayEquals(copy, bytes); - revertBytes = TransactionalKvBackendImpl.revertByteArray(revertBytes); - Assertions.assertArrayEquals(bytes, revertBytes); - } -} diff --git a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java index 7cff7bcc553..2f11a435f34 100644 --- a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java @@ -32,7 +32,6 @@ import org.apache.gravitino.Entity; import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; -import org.apache.gravitino.EntitySerDe; import org.apache.gravitino.EntityStore; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.Metalake; @@ -77,9 +76,6 @@ public void clear() { @Override public void initialize(Config config) throws RuntimeException {} - @Override - public void setSerDe(EntitySerDe entitySerDe) {} - @Override public List list( Namespace namespace, Class cl, EntityType entityType) throws IOException { @@ -250,7 +246,6 @@ public void testEntityStoreAndRetrieve() throws Exception { InMemoryEntityStore store = new InMemoryEntityStore(); store.initialize(Mockito.mock(Config.class)); - store.setSerDe(Mockito.mock(EntitySerDe.class)); store.put(metalake); store.put(catalog); diff --git a/docs/glossary.md b/docs/glossary.md index be06940c1f8..83e97d915aa 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -300,10 +300,6 @@ license: "This software is licensed under the Apache License version 2." - A set of rules and conventions for building and interacting with web services using standard HTTP methods. -## RocksDB - -- An open source key-value pair storage database. - ## Schema - A logical container for organizing tables in a database. diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 0810c7c3c63..452fda4f823 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -53,9 +53,7 @@ The following table lists the storage configuration items: | Configuration item | Description | Default value | Required | Since version | |---------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------|-------------------------------------------------|------------------| | `gravitino.entity.store` | Which entity storage implementation to use. Only`relational` storage is currently supported. | `relational` | No | 0.1.0 | -| `gravitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | | `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | -| `gravitino.entity.store.kv.deleteAfterTimeMs` | It is deprecated since Gravitino 0.5.0. Please use `gravitino.entity.store.deleteAfterTimeMs` instead. | `604800000`(7 days) | No | 0.3.0 | | `gravitino.entity.store.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.5.0 | | `gravitino.entity.store.versionRetentionCount` | The Count of versions allowed to be retained, including the current version, used to delete old versions data. Set to at least 1 and no greater than 10. | `1` | No | 0.5.0 | | `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `H2`, `MySQL` and `PostgreSQL` is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f1f29cf186d..6827c21cd95 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -50,7 +50,6 @@ commons-collections3 = "3.2.2" commons-configuration1 = "1.6" commons-dbcp2 = "2.11.0" caffeine = "2.9.3" -rocksdbjni = "7.10.2" iceberg = '1.5.2' # used for Gravitino Iceberg catalog and Iceberg REST service iceberg4spark = "1.4.1" # used for compile spark connector paimon = '0.8.0' @@ -185,7 +184,6 @@ commons-lang3 = { group = "org.apache.commons", name = "commons-lang3", version. commons-logging = { group = "commons-logging", name = "commons-logging", version.ref = "commons-logging" } commons-io = { group = "commons-io", name = "commons-io", version.ref = "commons-io" } caffeine = { group = "com.github.ben-manes.caffeine", name = "caffeine", version.ref = "caffeine" } -rocksdbjni = { group = "org.rocksdb", name = "rocksdbjni", version.ref = "rocksdbjni" } commons-collections4 = { group = "org.apache.commons", name = "commons-collections4", version.ref = "commons-collections4" } commons-collections3 = { group = "commons-collections", name = "commons-collections", version.ref = "commons-collections3" } commons-configuration1 = { group = "commons-configuration", name = "commons-configuration", version.ref = "commons-configuration1" } diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java index e20681c4070..d0ce1710d7f 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/MiniGravitino.java @@ -19,7 +19,6 @@ package org.apache.gravitino.integration.test; import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; @@ -32,7 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -201,8 +199,6 @@ public void stop() throws IOException, InterruptedException { restClient.close(); try { FileUtils.deleteDirectory(mockConfDir); - FileUtils.deleteDirectory( - FileUtils.getFile(serverConfig.get(ENTITY_KV_ROCKSDB_BACKEND_PATH))); } catch (Exception e) { // Ignore } @@ -241,8 +237,6 @@ private void customizeConfigFile(String configTempFileName, String configFileNam configMap.put( GravitinoServer.WEBSERVER_CONF_PREFIX + JettyServerConfig.WEBSERVER_HTTP_PORT.getKey(), String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); - configMap.put( - Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH.getKey(), "/tmp/gravitino-" + UUID.randomUUID()); configMap.putAll(getIcebergRestServiceConfigs()); configMap.putAll(context.customConfig); diff --git a/meta/build.gradle.kts b/meta/build.gradle.kts deleted file mode 100644 index a02ebbbb08e..00000000000 --- a/meta/build.gradle.kts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - `maven-publish` - id("java") - id("idea") - alias(libs.plugins.protobuf) -} - -dependencies { - implementation(libs.protobuf.java) -} - -sourceSets { - main { - proto.srcDir("meta/src/main/proto") - resources.srcDir("meta/src/main/resources") - } -} - -protobuf { - protoc { artifact = "com.google.protobuf:protoc:${libs.versions.protoc.get()}" } -} diff --git a/meta/src/main/proto/gravitino_meta.proto b/meta/src/main/proto/gravitino_meta.proto deleted file mode 100644 index de0a56ce70c..00000000000 --- a/meta/src/main/proto/gravitino_meta.proto +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -syntax = "proto3"; - -import "google/protobuf/timestamp.proto"; - -option java_multiple_files = true; -option java_package = "org.apache.gravitino.proto"; - -/** - * The version of schema definition. Schema is backward compatible within the - * same major version. The minor version is used for schema evolution. - */ -message SchemaVersion { - uint32 major_number = 1; - uint32 minor_number = 2; -} - -/** - * The AuditInfo message is used to record the audit information of a resource. - */ -message AuditInfo { - optional string creator = 1; - optional google.protobuf.Timestamp create_time = 2; - optional string last_modifier = 3; - optional google.protobuf.Timestamp last_modified_time = 4; -} - -/** - * The Metalake message is used to record the Metalake information. Metalake is used track all - the metadata of the data lake, data warehouse, and data mart. - */ -message Metalake { - uint64 id = 1; - string name = 2; - optional string comment = 3; - map properties = 4; - AuditInfo audit_info = 5; - SchemaVersion version = 6; -} - -message Catalog { - enum Type { - RELATIONAL = 0; // Catalog Type for Relational Data Structure, like db.table. - FILESET = 1; // Catalog Type for File System (including HDFS, S3, etc.), like path/to/file. - MESSAGING = 2; // Catalog Type for MESSAGING, like Kafka://topic. - } - - uint64 id = 1; - string name = 2; - Type type = 3; - optional string comment = 4; - map properties = 5; - AuditInfo audit_info = 6; - string provider = 7; -} - -message Schema { - uint64 id = 1; - string name = 2; - AuditInfo audit_info = 3; - // These two fields will be used when the schema object is managed by Gravitino, otherwise, it - // will be ignored. - optional string comment = 4; - map properties = 5; -} - -message Table { - uint64 id = 1; - string name = 2; - AuditInfo audit_info = 3; -} - -/** - * The Fileset message is used to record the fileset object information. Fileset object represents - * the non-tabular data on FS-like storage, like HDFS, S3, etc. - */ -message Fileset { - enum Type { - MANAGED = 0; // The storage location is managed by Gravitino. When specified, the data will be - // deleted when the file object is deleted. - EXTERNAL = 1; // The storage location is managed by user. When specified, the data will not be - // deleted when the file object is deleted. - } - - uint64 id = 1; - string name = 2; - optional string comment = 3; - Type type = 4; - string storage_location = 5; - map properties = 6; - AuditInfo audit_info = 7; -} - -message Topic { - uint64 id = 1; - string name = 2; - optional string comment = 3; - map properties = 4; - AuditInfo audit_info = 5; -} - -message User { - uint64 id = 1; - string name = 2; - repeated string role_names = 3; - repeated uint64 role_ids = 4; - AuditInfo audit_info = 5; -} - -message Group { - uint64 id = 1; - string name = 2; - repeated string role_names = 3; - repeated uint64 role_ids = 4; - AuditInfo audit_info = 5; -} - -message SecurableObject { - string full_name = 1; - string type = 2; - repeated string privilege_names = 3; - repeated string privilege_conditions = 4; -} - -message Role { - uint64 id = 1; - string name = 2; - repeated SecurableObject securable_objects = 3; - map properties = 4; - AuditInfo audit_info = 5; -} diff --git a/rfc/rfc-2/Entity-key-encoding-for-kv-store.md b/rfc/rfc-2/Entity-key-encoding-for-kv-store-deprecated.md similarity index 96% rename from rfc/rfc-2/Entity-key-encoding-for-kv-store.md rename to rfc/rfc-2/Entity-key-encoding-for-kv-store-deprecated.md index 7de06eb1af0..ebd75d30868 100644 --- a/rfc/rfc-2/Entity-key-encoding-for-kv-store.md +++ b/rfc/rfc-2/Entity-key-encoding-for-kv-store-deprecated.md @@ -17,13 +17,19 @@ under the License. --> -# RFC-1: Entity Key Encoding design for KV store +# RFC-1: Entity Key Encoding design for KV store (deprecated) | Revision | Owner | Date | | :------- |-------| ------| | v0.1 | Qi Yu | 1/8/2023| | v0.2 | Qi Yu | 5/9/2023| +## ⚠️ DEPRECATED + +This document is no longer maintained and may contain outdated information. + +Deprecated on: 2024-11 + ## Background Currently, there will be many data to storage, for example, User information. Such as username, password, user property, this data is structure data diff --git a/rfc/rfc-3/Transaction-implementation-on-kv.md b/rfc/rfc-3/Transaction-implementation-on-kv.md deleted file mode 100644 index 4d902700b38..00000000000 --- a/rfc/rfc-3/Transaction-implementation-on-kv.md +++ /dev/null @@ -1,140 +0,0 @@ - - -# RFC-1: Design of transaction mechanism on key-value store - -| Revision | Owner | Date | -| :------- |-------|------------| -| v0.1 | Qi Yu | 21/11/2023 | - -## Background - -Currently, our storage layer heavily relies on the transaction mechanism provided by key-value storage backend such as RocksDB to ensure reliability. However, some key-value pair databases do not support transaction operations, making it challenging for Apache Gravitino to adapt to other KV databases such as Redis, Cassandra, Hbase, and so on. - -To make gravitino adapt different key-value stores, we need to eliminate transactional dependency and come up with alternative solutions. - -## Goal - -1. Support transaction operations on key-value store -2. Do not make too much change to the current storage layer -3. Easy to implement and maintain. - -## Design - -Based on [previous design](../rfc-2/Entity-key-encoding-for-kv-store.md), to support transaction, we use MVCC to implement transaction mechanism. The main idea is to add a version number to each key-value pair. When a transaction is committed, we will update the version number of the key-value pair. When a transaction is aborted, we will delete the key-value pair. -The following figure shows the design of transaction mechanism. - -![](../resouces/Transaction.jpg) - -Note: -1. A, B, C, is the original keys -2. ts1, ts2, ts3 are the timestamp of the transaction -3. tx_ts1, tx_ts2, tx_ts3 are commit marks. - -Take the above figure as an example, the transaction is as follows: -Assuming we want to update three key-values pairs: -1. Change the value of Key A from 100 to 150 -2. Change the value of Key B from 150 to 200 -3. Change the value of Key C from 300 to 200 - -The transaction process is as follows: - -**Prepare stage** -1. First, we will get a TSO(Timestamp oracle) named ts2 as the transaction id. -2. Then we will read old values of A, B, C using the TSO ts2. -3. Update the value of A, B, C to 150, 200, 200 respectively. -4. Then we will add a version number to each key-value pair, the version number is the transaction id. -5. Then we will write the new key-value pairs to the storage layer. - -**Commit stage** -We would construct the commit mark tx_ts2 and save it to the storage layer. The value of tx_ts2 contains -all keys in the transaction. - -**Rollback stage** -If any steps in prepare stage or commit stage fails, we will roll back the transaction. In fact, we -need to do nothing to roll back the transaction. Because we have added a version number to each key-value pair and -those keys in the failed transaction will not be visible to users. - - -## Detail implementation - -### Write process - -Assuming we want to put key-value pairs 'key1:value1' and 'key2:value2' to the storage layer. The tso we get is 1, The process is as follows: - -In prepare stage, we will put the following keys into the storage layer: -``` -KEY: 0x6B657931 1F FFFFFFFFFFFFFFFE ---> VALUE: 0x00000000 00000000 76616C756531 -KEY: 0x6B657932 1F FFFFFFFFFFFFFFFE ---> VALUE: 0x00000000 00000000 76616C756532 -``` -- `6B657931`: the hex of 'key1' -- `1F`: we use `1F` space as the separator of key and TSO. -- `FFFFFFFFFFFFFFFE`: Represent tso 1. Note, we use 8 bytes to represent tso. How to convert 1 to `FFFFFFFFFFFFFFFE`? First we convert 1 to 0x0000000000000001, then we use `^` 0xFF to get the complement of 0x0000000000000001, then we get 0xFFFFFFFFFFFFFFFE. - Why we need to do So? we want to make sure that the key-value pair with the same key and different tso can be sorted in descending order. that is: if tso1 >= tso2, then binary(tso1) <= binary(tso2). -- `00000000 00000000 76616C756531`: the hex format of 'value1'. The first byte `0x00` means is put operation, if the value is '0x01', it marks the key-value pair has been deleted. - -In the commit stage, we will put the following keys into the storage layer: -``` -KEY: 0x1E 1F FFFFFFFFFFFFFFFE ---> VALUE: keys of the transaction involved. -``` -- `1E`: prefix of transaction commit mark -- `1F`: separator -- `FFFFFFFFFFFFFFFE`: Represent tso 1. Note, we use 8 bytes to represent tso. How to convert 1 to `FFFFFFFFFFFFFFFE`? First we convert 1 to 0x0000000000000001, then we use `^` 0xFF to get the complement of 0x0000000000000001, then we get 0xFFFFFFFFFFFFFFFE. - -### Read process - -Assuming we have the following key-value pairs in storage layer: -``` -KEY: 0x6B657931 1F FFFFFFFFFFFFFFFD ---> VALUE: 0x00000000 00000000 76616C756532 -KEY: 0x6B657931 1F FFFFFFFFFFFFFFFE ---> VALUE: 0x00000000 00000000 76616C756531 -KEY: 0x1E 1F FFFFFFFFFFFFFFFD ---> VALUE: keys of the transaction involved. -KEY: 0x1E 1F FFFFFFFFFFFFFFFE ---> VALUE: keys of the transaction involved. -``` -If we want to get the value of key1, the process is as follows: -1. First, we will get a TSO(Timestamp oracle) 3 as the transaction id. -2. Construct the start key to scan the storage layer: `0x6B657931 1F FFFFFFFFFFFFFFFC` -3. Find the first key that with prefix `0x6B657931` and greater than start key `0x6B657931 1F FFFFFFFFFFFFFFFC`, we will get `0x6B657931 1F FFFFFFFFFFFFFFFD` -4. If there exists the key `0x1E 1F FFFFFFFFFFFFFFFE`, means the transaction is committed, then we will get the value of key1 from `0x6B657931 1F FFFFFFFFFFFFFFFD`. -5. or repeat steps 3 -6. If the value of key `0x6B657931 1F FFFFFFFFFFFFFFFD` starts will `0x00`, then we will get the value of key1 from `0x6B657931 1F FFFFFFFFFFFFFFFD`. if the value of key `0x6B657931 1F FFFFFFFFFFFFFFFD` starts will `0x01`, it means this key-value pair has been removed, so we return null. - -### Delete process -Delete steps are almost the same as that of write process except that the prefix of it is `0x01` instead of `0x00`. - -Put key 'test1': -``` -KEY: 0x6B657931 1F FFFFFFFFFFFFFFFE ---> VALUE: 0x00000000 00000000 76616C756531 -``` - -Delete key 'test1': -``` -KEY: 0x6B657931 1F FFFFFFFFFFFFFFFE ---> VALUE: 0x01000000 00000000 76616C756531 -``` - -### Scan and range query process -Scan and range query are almost the same as that of read process, for more detailed information, please see related implementation `TransactionalKvBackendImpl`. - -## Key format after this modification - -- Keys that start with 0x'1D0000' store the contents of id-name mapping. for more please refer to class `KvNameMappingService`. -- Keys that start with 0x'1D0001' store the data of current timestamp which is used for generating transaction id, for more please refer to class `TransactionIdGeneratorImpl`. -- Keys that start with 0x'1D0002' store the information of storage layout version. For more please refer to `KvEntityStore#initStorageVersionInfo` -- Keys that start with 0x'1D0003' store tha transaction id that was used by `KvGarbageCollector` last time. -- Keys that start with 0x'1E' store transaction marks which mark the transaction is committed or not. -- Other key spaces are used to store gravitino entities like `metalakes`,`catalogs`, `scheams`, `tables` and so on. it usually starts with from 0x'20'(space) to 0x'7F'(delete). For more please refer to class `KvEntityStoreImpl`. \ No newline at end of file diff --git a/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java b/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java index c11e48f3b12..428d6424912 100644 --- a/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java +++ b/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.server; -import static org.apache.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -26,7 +25,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.UUID; import org.apache.commons.io.FileUtils; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.auxiliary.AuxiliaryServiceManager; @@ -34,7 +32,6 @@ import org.apache.gravitino.server.web.JettyServerConfig; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -46,8 +43,6 @@ public class TestGravitinoServer { private GravitinoServer gravitinoServer; - private final String ROCKS_DB_STORE_PATH = - "/tmp/gravitino_test_server_" + UUID.randomUUID().toString().replace("-", ""); private ServerConfig spyServerConfig; @BeforeAll @@ -55,8 +50,6 @@ void initConfig() throws IOException { ServerConfig serverConfig = new ServerConfig(); serverConfig.loadFromMap( ImmutableMap.of( - ENTITY_KV_ROCKSDB_BACKEND_PATH.getKey(), - ROCKS_DB_STORE_PATH, GravitinoServer.WEBSERVER_CONF_PREFIX + JettyServerConfig.WEBSERVER_HTTP_PORT.getKey(), String.valueOf(RESTUtils.findAvailablePort(5000, 6000))), t -> true); @@ -71,12 +64,6 @@ void initConfig() throws IOException { @BeforeEach public void setUp() { - // Remove rocksdb storage file if exists - try { - FileUtils.deleteDirectory(FileUtils.getFile(ROCKS_DB_STORE_PATH)); - } catch (Exception e) { - // Ignore - } gravitinoServer = new GravitinoServer(spyServerConfig, GravitinoEnv.getInstance()); } @@ -105,12 +92,6 @@ public void testInitialize() { gravitinoServer.initialize(); } - @Test - void testConfig() { - Assertions.assertEquals( - ROCKS_DB_STORE_PATH, spyServerConfig.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); - } - @Test public void testStartAndStop() throws Exception { gravitinoServer.initialize(); diff --git a/settings.gradle.kts b/settings.gradle.kts index a36fde93cd3..1bfade026cd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -25,7 +25,7 @@ rootProject.name = "gravitino" val scalaVersion: String = gradle.startParameter.projectProperties["scalaVersion"]?.toString() ?: settings.extra["defaultScalaVersion"].toString() -include("api", "common", "core", "meta", "server", "server-common") +include("api", "common", "core", "server", "server-common") include("catalogs:catalog-common") include("catalogs:catalog-hive") include("catalogs:hive-metastore-common")