From 76243d41dec71f95b2dae38f94bceae8f878489a Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 5 Feb 2024 20:59:24 +0800 Subject: [PATCH] fix comments --- .../com/datastrato/gravitino/Configs.java | 17 +-- .../gravitino/EntityStoreFactory.java | 6 +- .../datastrato/gravitino/GravitinoEnv.java | 18 --- .../gravitino/storage/kv/KvEntityStore.java | 2 + .../RelationalBackend.java} | 12 +- .../RelationalEntityStore.java} | 42 +++---- .../mysql/MySQLBackend.java | 20 ++-- .../datastrato/gravitino/TestEntityStore.java | 4 +- .../relational/TestRelationalEntityStore.java | 104 ++++++++++++++++++ 9 files changed, 158 insertions(+), 67 deletions(-) rename core/src/main/java/com/datastrato/gravitino/storage/{relation/RelationBackend.java => relational/RelationalBackend.java} (89%) rename core/src/main/java/com/datastrato/gravitino/storage/{relation/RelationEntityStore.java => relational/RelationalEntityStore.java} (72%) rename core/src/main/java/com/datastrato/gravitino/storage/{relation => relational}/mysql/MySQLBackend.java (75%) create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 850efb82682..3903870ca11 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -5,20 +5,22 @@ package com.datastrato.gravitino; import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; import java.io.File; +import org.apache.commons.lang3.StringUtils; public interface Configs { String DEFAULT_ENTITY_STORE = "kv"; - String RELATION_ENTITY_STORE = "rel"; + String RELATIONAL_ENTITY_STORE = "relational"; String ENTITY_STORE_KEY = "gravitino.entity.store"; String DEFAULT_ENTITY_KV_STORE = "RocksDBKvBackend"; String ENTITY_KV_STORE_KEY = "gravitino.entity.store.kv"; - String DEFAULT_ENTITY_RELATION_STORE = "MySQLBackend"; - String ENTITY_RELATION_STORE_KEY = "gravitino.entity.store.rel"; + String DEFAULT_ENTITY_RELATIONAL_STORE = "MySQLBackend"; + String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; @@ -49,12 +51,13 @@ public interface Configs { .stringConf() .createWithDefault(DEFAULT_ENTITY_KV_STORE); - ConfigEntry ENTITY_RELATION_STORE = - new ConfigBuilder(ENTITY_RELATION_STORE_KEY) - .doc("Detailed implementation of relation storage") + ConfigEntry ENTITY_RELATIONAL_STORE = + new ConfigBuilder(ENTITY_RELATIONAL_STORE_KEY) + .doc("Detailed implementation of relational storage") .version("0.5.0") .stringConf() - .createWithDefault(DEFAULT_ENTITY_RELATION_STORE); + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .createWithDefault(DEFAULT_ENTITY_RELATIONAL_STORE); ConfigEntry ENTRY_KV_ROCKSDB_BACKEND_PATH = new ConfigBuilder(ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY) diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java index 1a5d010e821..dfc083e7e64 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java @@ -5,7 +5,7 @@ package com.datastrato.gravitino; import com.datastrato.gravitino.storage.kv.KvEntityStore; -import com.datastrato.gravitino.storage.relation.RelationEntityStore; +import com.datastrato.gravitino.storage.relational.RelationalEntityStore; import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,8 +24,8 @@ public class EntityStoreFactory { ImmutableMap.of( Configs.DEFAULT_ENTITY_STORE, KvEntityStore.class.getCanonicalName(), - Configs.RELATION_ENTITY_STORE, - RelationEntityStore.class.getCanonicalName()); + Configs.RELATIONAL_ENTITY_STORE, + RelationalEntityStore.class.getCanonicalName()); // Private constructor to prevent instantiation of this factory class. private EntityStoreFactory() {} diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index 6486641ceb0..9b27f561b09 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.metrics.source.JVMMetricsSource; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.RandomIdGenerator; -import com.datastrato.gravitino.storage.relation.RelationEntityStore; import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,8 +27,6 @@ public class GravitinoEnv { private EntityStore entityStore; - private EntitySerDe entitySerDe; - private CatalogManager catalogManager; private CatalogOperationDispatcher catalogOperationDispatcher; @@ -69,15 +66,9 @@ public void initialize(Config config) { this.metricsSystem = new MetricsSystem(); metricsSystem.register(new JVMMetricsSource()); - // Initialize EntitySerDe - this.entitySerDe = EntitySerDeFactory.createEntitySerDe(config); - // Initialize EntityStore this.entityStore = EntityStoreFactory.createEntityStore(config); entityStore.initialize(config); - if (!(entityStore instanceof RelationEntityStore)) { - entityStore.setSerDe(entitySerDe); - } // create and initialize a random id generator this.idGenerator = new RandomIdGenerator(); @@ -106,15 +97,6 @@ public Config config() { return config; } - /** - * Get the EntitySerDe associated with the Gravitino environment. - * - * @return The EntitySerDe instance. - */ - public EntitySerDe entitySerDe() { - return entitySerDe; - } - /** * Get the EntityStore associated with the Gravitino environment. * diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 9e9a355226c..18deb7a4cca 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -17,6 +17,7 @@ import com.datastrato.gravitino.Entity.EntityType; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.EntitySerDe; +import com.datastrato.gravitino.EntitySerDeFactory; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; @@ -103,6 +104,7 @@ public void initialize(Config config) throws RuntimeException { kvGarbageCollector.start(); this.storageLayoutVersion = initStorageVersionInfo(); + this.serDe = EntitySerDeFactory.createEntitySerDe(config); } @Override diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java similarity index 89% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java index 17a20a98088..8969db9f912 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java @@ -2,7 +2,7 @@ * Copyright 2024 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation; +package com.datastrato.gravitino.storage.relational; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Entity; @@ -17,17 +17,17 @@ import java.util.function.Function; /** Interface defining the operations for a Relation Backend. */ -public interface RelationBackend extends Closeable { +public interface RelationalBackend extends Closeable { /** - * Initializes the Relation Backend environment with the provided configuration. + * Initializes the Relational Backend environment with the provided configuration. * * @param config The configuration for the backend. */ void initialize(Config config); /** - * List the entities associated with the given parent namespace and entityType + * Lists the entities associated with the given parent namespace and entityType * * @param namespace The parent namespace of these entities. * @param entityType The type of these entities. @@ -40,7 +40,7 @@ List list(Namespace namespace, Entity.Enti throws NoSuchEntityException; /** - * Check the entity associated with the given identifier and entityType whether exists. + * Checks the entity associated with the given identifier and entityType whether exists. * * @param ident The identifier of the entity. * @param entityType The type of the entity. @@ -59,7 +59,7 @@ void insert(E e, boolean overwritten) throws EntityAlreadyExistsException; /** - * Update the entity. + * Updates the entity. * * @param ident The identifier of the entity which need be stored. * @param entityType The type of the entity. diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java similarity index 72% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java index 72b4aebc224..395d44f38f0 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java @@ -2,9 +2,9 @@ * Copyright 2024 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation; +package com.datastrato.gravitino.storage.relational; -import static com.datastrato.gravitino.Configs.ENTITY_RELATION_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; @@ -17,7 +17,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.storage.relation.mysql.MySQLBackend; +import com.datastrato.gravitino.storage.relational.mysql.MySQLBackend; import com.datastrato.gravitino.utils.Executable; import com.google.common.collect.ImmutableMap; import java.io.IOException; @@ -30,41 +30,43 @@ /** * Relation store to store entities. This means we can store entities in a relational store. I.e., * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link - * RelationBackend} interface + * RelationalBackend} interface */ -public class RelationEntityStore implements EntityStore { - private static final Logger LOGGER = LoggerFactory.getLogger(RelationEntityStore.class); - public static final ImmutableMap RELATION_BACKENDS = - ImmutableMap.of(Configs.DEFAULT_ENTITY_RELATION_STORE, MySQLBackend.class.getCanonicalName()); - private RelationBackend backend; +public class RelationalEntityStore implements EntityStore { + private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); + public static final ImmutableMap RELATIONAL_BACKENDS = + ImmutableMap.of( + Configs.DEFAULT_ENTITY_RELATIONAL_STORE, MySQLBackend.class.getCanonicalName()); + private RelationalBackend backend; @Override public void initialize(Config config) throws RuntimeException { this.backend = createRelationEntityBackend(config); } - private static RelationBackend createRelationEntityBackend(Config config) { - String backendName = config.get(ENTITY_RELATION_STORE); - String className = RELATION_BACKENDS.getOrDefault(backendName, backendName); + private static RelationalBackend createRelationEntityBackend(Config config) { + String backendName = config.get(ENTITY_RELATIONAL_STORE); + String className = RELATIONAL_BACKENDS.getOrDefault(backendName, backendName); if (Objects.isNull(className)) { throw new RuntimeException("Unsupported backend type..." + backendName); } try { - RelationBackend relationBackend = - (RelationBackend) Class.forName(className).getDeclaredConstructor().newInstance(); - relationBackend.initialize(config); - return relationBackend; + RelationalBackend relationalBackend = + (RelationalBackend) Class.forName(className).getDeclaredConstructor().newInstance(); + relationalBackend.initialize(config); + return relationalBackend; } catch (Exception e) { - LOGGER.error("Failed to create and initialize RelationBackend by name '{}'.", backendName, e); + LOGGER.error( + "Failed to create and initialize RelationalBackend by name '{}'.", backendName, e); throw new RuntimeException( - "Failed to create and initialize RelationBackend by name: " + backendName, e); + "Failed to create and initialize RelationalBackend by name: " + backendName, e); } } @Override public void setSerDe(EntitySerDe entitySerDe) { - throw new UnsupportedOperationException("Unsupported operation in relation entity store."); + throw new UnsupportedOperationException("Unsupported operation in relational entity store."); } @Override @@ -110,7 +112,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea @Override public R executeInTransaction(Executable executable) { - throw new UnsupportedOperationException("Unsupported operation in relation entity store."); + throw new UnsupportedOperationException("Unsupported operation in relational entity store."); } @Override diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java similarity index 75% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java index 355fc2d2a81..4b793a02ddc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java @@ -2,7 +2,7 @@ * Copyright 2024 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation.mysql; +package com.datastrato.gravitino.storage.relational.mysql; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Entity; @@ -11,23 +11,21 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.storage.relation.RelationBackend; +import com.datastrato.gravitino.storage.relational.RelationalBackend; import java.io.IOException; import java.util.List; import java.util.function.Function; /** - * {@link MySQLBackend} is a MySQL implementation of RelationBackend interface. If we want to use - * another relation implementation, We can just implement {@link RelationBackend} interface and use - * it in the Gravitino. + * {@link MySQLBackend} is a MySQL implementation of RelationalBackend interface. If we want to use + * another relational implementation, We can just implement {@link RelationalBackend} interface and + * use it in the Gravitino. */ -public class MySQLBackend implements RelationBackend { +public class MySQLBackend implements RelationalBackend { /** Initialize the MySQL backend instance. */ @Override - public void initialize(Config config) { - throw new UnsupportedOperationException("Unsupported operation now."); - } + public void initialize(Config config) {} @Override public List list( @@ -65,7 +63,5 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea } @Override - public void close() throws IOException { - throw new UnsupportedOperationException("Unsupported operation now."); - } + public void close() throws IOException {} } diff --git a/core/src/test/java/com/datastrato/gravitino/TestEntityStore.java b/core/src/test/java/com/datastrato/gravitino/TestEntityStore.java index 4ddb8bfcab1..6393854df2d 100644 --- a/core/src/test/java/com/datastrato/gravitino/TestEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/TestEntityStore.java @@ -45,7 +45,9 @@ public void clear() { } @Override - public void initialize(Config config) throws RuntimeException {} + public void initialize(Config config) throws RuntimeException { + this.serde = Mockito.mock(EntitySerDe.class); + } @Override public void setSerDe(EntitySerDe entitySerDe) { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java new file mode 100644 index 00000000000..039e3bf0f3a --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -0,0 +1,104 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational; + +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.EntityStoreFactory; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.meta.BaseMetalake; +import java.io.IOException; +import java.util.function.Function; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestRelationalEntityStore { + private static EntityStore store; + + @BeforeAll + public static void setUp() { + Config config = Mockito.mock(Config.class); + Mockito.when(config.get(Configs.ENTITY_STORE)).thenReturn(Configs.RELATIONAL_ENTITY_STORE); + Mockito.when(config.get(Configs.ENTITY_RELATIONAL_STORE)) + .thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + store = EntityStoreFactory.createEntityStore(config); + store.initialize(config); + Assertions.assertTrue(store instanceof RelationalEntityStore); + } + + @AfterAll + public static void teardown() throws IOException { + store.close(); + store = null; + } + + @Test + public void testSetSerDe() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> store.setSerDe(null)); + } + + @Test + public void testExecuteInTransaction() { + Assertions.assertThrows( + UnsupportedOperationException.class, () -> store.executeInTransaction(null)); + } + + @Test + public void testExists() throws IOException { + NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> store.exists(nameIdentifier, Entity.EntityType.METALAKE)); + } + + @Test + public void testPut() throws IOException { + BaseMetalake metalake = Mockito.mock(BaseMetalake.class); + Assertions.assertThrows(UnsupportedOperationException.class, () -> store.put(metalake, false)); + } + + @Test + public void testGet() { + NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> store.get(nameIdentifier, Entity.EntityType.METALAKE, BaseMetalake.class)); + } + + @Test + public void testUpdate() { + NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); + Function function = Mockito.mock(Function.class); + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> + store.update(nameIdentifier, BaseMetalake.class, Entity.EntityType.METALAKE, function)); + } + + @Test + public void testList() { + Namespace namespace = Mockito.mock(Namespace.class); + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> store.list(namespace, BaseMetalake.class, Entity.EntityType.METALAKE)); + } + + @Test + public void testDelete() { + NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> store.delete(nameIdentifier, Entity.EntityType.METALAKE, false)); + } +}