From 4432c857113e5303323150e6af2fe14fbfe90c21 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Fri, 2 Feb 2024 11:43:49 +0800 Subject: [PATCH 01/34] add rel interface --- .../storage/relation/RelationBackend.java | 41 +++++++ .../storage/relation/RelationEntityStore.java | 111 ++++++++++++++++++ .../storage/relation/mysql/MysqlBackend.java | 67 +++++++++++ 3 files changed, 219 insertions(+) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MysqlBackend.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java new file mode 100644 index 00000000000..43c1af04b1d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relation; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.function.Function; + +public interface RelationBackend extends Closeable { + + void initialize(Config config) throws IOException; + + List list(Namespace namespace, Entity.EntityType entityType) + throws IOException; + + boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException; + + void put(E e, boolean overwritten) + throws IOException, EntityAlreadyExistsException; + + E update( + NameIdentifier ident, Entity.EntityType entityType, Function updater) + throws IOException, NoSuchEntityException, AlreadyExistsException; + + E get(NameIdentifier ident, Entity.EntityType entityType) + throws NoSuchEntityException, IOException; + + boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) + throws IOException; +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java new file mode 100644 index 00000000000..a8a62b449f5 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java @@ -0,0 +1,111 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relation; + +import static com.datastrato.gravitino.Configs.ENTITY_RELATION_STORE; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.EntitySerDe; +import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.storage.relation.mysql.MysqlBackend; +import com.datastrato.gravitino.utils.Executable; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RelationEntityStore implements EntityStore { + private static final Logger LOGGER = LoggerFactory.getLogger(RelationEntityStore.class); + public static final ImmutableMap RELATION_BACKENDS = + ImmutableMap.of("MysqlBackend", MysqlBackend.class.getCanonicalName()); + private RelationBackend backend; + + @Override + public void initialize(Config config) throws RuntimeException { + this.backend = createRelationEntityBackend(config); + } + + private static RelationBackend createRelationEntityBackend(Config config) { + String backendName = config.get(ENTITY_RELATION_STORE); + String className = RELATION_BACKENDS.getOrDefault(backendName, backendName); + if (Objects.isNull(className)) { + throw new RuntimeException("Unsupported backend type..." + backendName); + } + + try { + RelationBackend relationBackend = + (RelationBackend) Class.forName(className).getDeclaredConstructor().newInstance(); + relationBackend.initialize(config); + return relationBackend; + } catch (Exception e) { + LOGGER.error("Failed to create and initialize RelationBackend by name '{}'.", backendName, e); + throw new RuntimeException( + "Failed to create and initialize RelationBackend by name: " + backendName, e); + } + } + + @Override + public void setSerDe(EntitySerDe entitySerDe) { + throw new UnsupportedOperationException("Unsupported operation in relation entity store."); + } + + @Override + public List list( + Namespace namespace, Class type, Entity.EntityType entityType) throws IOException { + return backend.list(namespace, entityType); + } + + @Override + public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException { + return backend.exists(ident, entityType); + } + + @Override + public void put(E e, boolean overwritten) + throws IOException, EntityAlreadyExistsException { + backend.put(e, overwritten); + } + + @Override + public E update( + NameIdentifier ident, Class type, Entity.EntityType entityType, Function updater) + throws IOException, NoSuchEntityException, AlreadyExistsException { + return backend.update(ident, entityType, updater); + } + + @Override + public E get( + NameIdentifier ident, Entity.EntityType entityType, Class e) + throws NoSuchEntityException, IOException { + return backend.get(ident, entityType); + } + + @Override + public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) + throws IOException { + return backend.delete(ident, entityType, cascade); + } + + @Override + public R executeInTransaction(Executable executable) + throws E, IOException { + throw new UnsupportedOperationException("Unsupported operation in relation entity store."); + } + + @Override + public void close() throws IOException { + backend.close(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MysqlBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MysqlBackend.java new file mode 100644 index 00000000000..508f81a83ce --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MysqlBackend.java @@ -0,0 +1,67 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relation.mysql; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.storage.relation.RelationBackend; +import java.io.IOException; +import java.util.List; +import java.util.function.Function; + +public class MysqlBackend implements RelationBackend { + @Override + public void initialize(Config config) throws IOException { + throw new UnsupportedOperationException("Unsupported operation now."); + } + + @Override + public List list( + Namespace namespace, Entity.EntityType entityType) throws IOException { + throw new UnsupportedOperationException("Unsupported operation now."); + } + + @Override + public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException { + throw new UnsupportedOperationException("Unsupported operation now."); + } + + @Override + public void put(E e, boolean overwritten) + throws IOException, EntityAlreadyExistsException { + throw new UnsupportedOperationException("Unsupported operation now."); + } + + @Override + public E update( + NameIdentifier ident, Entity.EntityType entityType, Function updater) + throws IOException, NoSuchEntityException, AlreadyExistsException { + throw new UnsupportedOperationException("Unsupported operation now."); + } + + @Override + public E get( + NameIdentifier ident, Entity.EntityType entityType) + throws NoSuchEntityException, IOException { + throw new UnsupportedOperationException("Unsupported operation now."); + } + + @Override + public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) + throws IOException { + throw new UnsupportedOperationException("Unsupported operation now."); + } + + @Override + public void close() throws IOException { + throw new UnsupportedOperationException("Unsupported operation now."); + } +} From ca62a8a3442655f9a8bdb39c23048eeafa0c704f Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Sun, 4 Feb 2024 09:51:51 +0800 Subject: [PATCH 02/34] fix the comment --- .../storage/relation/RelationBackend.java | 2 +- .../storage/relation/RelationEntityStore.java | 15 ++++++++++----- .../{MysqlBackend.java => MySQLBackend.java} | 11 +++++++++-- 3 files changed, 20 insertions(+), 8 deletions(-) rename core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/{MysqlBackend.java => MySQLBackend.java} (84%) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java index 43c1af04b1d..79dfb7a2281 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java @@ -26,7 +26,7 @@ List list(Namespace namespace, Entity.Enti boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException; - void put(E e, boolean overwritten) + void insert(E e, boolean overwritten) throws IOException, EntityAlreadyExistsException; E update( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java index a8a62b449f5..1c3021c5bef 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java @@ -7,6 +7,7 @@ import static com.datastrato.gravitino.Configs.ENTITY_RELATION_STORE; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.EntitySerDe; @@ -16,7 +17,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.storage.relation.mysql.MysqlBackend; +import com.datastrato.gravitino.storage.relation.mysql.MySQLBackend; import com.datastrato.gravitino.utils.Executable; import com.google.common.collect.ImmutableMap; import java.io.IOException; @@ -26,10 +27,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Relation store to store entities. This means we can store entities in a relational store. I.e., + * MYSQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link + * RelationBackend} interface + */ public class RelationEntityStore implements EntityStore { private static final Logger LOGGER = LoggerFactory.getLogger(RelationEntityStore.class); public static final ImmutableMap RELATION_BACKENDS = - ImmutableMap.of("MysqlBackend", MysqlBackend.class.getCanonicalName()); + ImmutableMap.of(Configs.DEFAULT_ENTITY_RELATION_STORE, MySQLBackend.class.getCanonicalName()); private RelationBackend backend; @Override @@ -75,7 +81,7 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws @Override public void put(E e, boolean overwritten) throws IOException, EntityAlreadyExistsException { - backend.put(e, overwritten); + backend.insert(e, overwritten); } @Override @@ -99,8 +105,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea } @Override - public R executeInTransaction(Executable executable) - throws E, IOException { + public R executeInTransaction(Executable executable) { throw new UnsupportedOperationException("Unsupported operation in relation entity store."); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MysqlBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java similarity index 84% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MysqlBackend.java rename to core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java index 508f81a83ce..d1fd045b1c7 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MysqlBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java @@ -17,7 +17,14 @@ import java.util.List; import java.util.function.Function; -public class MysqlBackend implements RelationBackend { +/** + * {@link MySQLBackend} is a MySQL implementation of RelationBackend interface. If we want to use + * another relation implementation, We can just implement {@link RelationBackend} interface and use + * it in the Gravitino. + */ +public class MySQLBackend implements RelationBackend { + + /** Initialize the MySQL backend instance. */ @Override public void initialize(Config config) throws IOException { throw new UnsupportedOperationException("Unsupported operation now."); @@ -35,7 +42,7 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws } @Override - public void put(E e, boolean overwritten) + public void insert(E e, boolean overwritten) throws IOException, EntityAlreadyExistsException { throw new UnsupportedOperationException("Unsupported operation now."); } From b29b0b9da28aa1a218659ae3b48b062fd4fe440a Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Sun, 4 Feb 2024 10:26:55 +0800 Subject: [PATCH 03/34] add the java doc for RelationBackend --- .../storage/relation/RelationBackend.java | 71 ++++++++++++++++--- .../storage/relation/RelationEntityStore.java | 8 ++- .../storage/relation/mysql/MySQLBackend.java | 17 ++--- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java index 79dfb7a2281..17a20a98088 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java @@ -10,32 +10,85 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import java.io.Closeable; import java.io.IOException; import java.util.List; import java.util.function.Function; +/** Interface defining the operations for a Relation Backend. */ public interface RelationBackend extends Closeable { - void initialize(Config config) throws IOException; + /** + * Initializes the Relation Backend environment with the provided configuration. + * + * @param config The configuration for the backend. + */ + void initialize(Config config); + /** + * List the entities associated with the given parent namespace and entityType + * + * @param namespace The parent namespace of these entities. + * @param entityType The type of these entities. + * @return The list of entities associated with the given parent namespace and entityType, or null + * if the entities does not exist. + * @throws NoSuchEntityException If the corresponding parent entity of these list entities cannot + * be found. + */ List list(Namespace namespace, Entity.EntityType entityType) - throws IOException; + throws NoSuchEntityException; - boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException; + /** + * Check the entity associated with the given identifier and entityType whether exists. + * + * @param ident The identifier of the entity. + * @param entityType The type of the entity. + * @return True, if the entity can be found, else return false. + */ + boolean exists(NameIdentifier ident, Entity.EntityType entityType); + /** + * Stores the entity, possibly overwriting an existing entity if specified. + * + * @param e The entity which need be stored. + * @param overwritten If true, overwrites the existing value. + * @throws EntityAlreadyExistsException If the entity already exists and overwrite is false. + */ void insert(E e, boolean overwritten) - throws IOException, EntityAlreadyExistsException; + throws EntityAlreadyExistsException; + /** + * Update the entity. + * + * @param ident The identifier of the entity which need be stored. + * @param entityType The type of the entity. + * @return The entity after updating. + * @throws NoSuchEntityException If the entity is not exist. + */ E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, AlreadyExistsException; + throws NoSuchEntityException; + /** + * Retrieves the entity associated with the identifier and the entity type. + * + * @param ident The identifier of the entity. + * @param entityType The type of the entity. + * @return The entity associated with the identifier and the entity type, or null if the key does + * not exist. + * @throws IOException If an I/O exception occurs during retrieval. + */ E get(NameIdentifier ident, Entity.EntityType entityType) - throws NoSuchEntityException, IOException; - - boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) throws IOException; + + /** + * Deletes the entity associated with the identifier and the entity type. + * + * @param ident The identifier of the entity. + * @param entityType The type of the entity. + * @param cascade True, If you need to cascade delete entities, else false. + * @return True, if the entity was successfully deleted, else false. + */ + boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java index 1c3021c5bef..72b4aebc224 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java @@ -29,7 +29,7 @@ /** * Relation store to store entities. This means we can store entities in a relational store. I.e., - * MYSQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link + * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link * RelationBackend} interface */ public class RelationEntityStore implements EntityStore { @@ -95,7 +95,11 @@ public E update( public E get( NameIdentifier ident, Entity.EntityType entityType, Class e) throws NoSuchEntityException, IOException { - return backend.get(ident, entityType); + E entity = backend.get(ident, entityType); + if (entity == null) { + throw new NoSuchEntityException(ident.toString()); + } + return entity; } @Override diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java index d1fd045b1c7..355fc2d2a81 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java @@ -10,7 +10,6 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.storage.relation.RelationBackend; import java.io.IOException; @@ -26,44 +25,42 @@ public class MySQLBackend implements RelationBackend { /** Initialize the MySQL backend instance. */ @Override - public void initialize(Config config) throws IOException { + public void initialize(Config config) { throw new UnsupportedOperationException("Unsupported operation now."); } @Override public List list( - Namespace namespace, Entity.EntityType entityType) throws IOException { + Namespace namespace, Entity.EntityType entityType) throws NoSuchEntityException { throw new UnsupportedOperationException("Unsupported operation now."); } @Override - public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException { + public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { throw new UnsupportedOperationException("Unsupported operation now."); } @Override public void insert(E e, boolean overwritten) - throws IOException, EntityAlreadyExistsException { + throws EntityAlreadyExistsException { throw new UnsupportedOperationException("Unsupported operation now."); } @Override public E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, AlreadyExistsException { + throws NoSuchEntityException { throw new UnsupportedOperationException("Unsupported operation now."); } @Override public E get( - NameIdentifier ident, Entity.EntityType entityType) - throws NoSuchEntityException, IOException { + NameIdentifier ident, Entity.EntityType entityType) throws IOException { throw new UnsupportedOperationException("Unsupported operation now."); } @Override - public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) - throws IOException { + public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { throw new UnsupportedOperationException("Unsupported operation now."); } From 0f28556cb97fe67d9875235d6a05883131d20822 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 5 Feb 2024 20:59:24 +0800 Subject: [PATCH 04/34] fix comments --- .../storage/relation/RelationBackend.java | 94 -------------- .../storage/relation/RelationEntityStore.java | 120 ------------------ .../storage/relation/mysql/MySQLBackend.java | 71 ----------- 3 files changed, 285 deletions(-) delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java deleted file mode 100644 index 17a20a98088..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationBackend.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.storage.relation; - -import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.EntityAlreadyExistsException; -import com.datastrato.gravitino.HasIdentifier; -import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.function.Function; - -/** Interface defining the operations for a Relation Backend. */ -public interface RelationBackend extends Closeable { - - /** - * Initializes the Relation Backend environment with the provided configuration. - * - * @param config The configuration for the backend. - */ - void initialize(Config config); - - /** - * List the entities associated with the given parent namespace and entityType - * - * @param namespace The parent namespace of these entities. - * @param entityType The type of these entities. - * @return The list of entities associated with the given parent namespace and entityType, or null - * if the entities does not exist. - * @throws NoSuchEntityException If the corresponding parent entity of these list entities cannot - * be found. - */ - List list(Namespace namespace, Entity.EntityType entityType) - throws NoSuchEntityException; - - /** - * Check the entity associated with the given identifier and entityType whether exists. - * - * @param ident The identifier of the entity. - * @param entityType The type of the entity. - * @return True, if the entity can be found, else return false. - */ - boolean exists(NameIdentifier ident, Entity.EntityType entityType); - - /** - * Stores the entity, possibly overwriting an existing entity if specified. - * - * @param e The entity which need be stored. - * @param overwritten If true, overwrites the existing value. - * @throws EntityAlreadyExistsException If the entity already exists and overwrite is false. - */ - void insert(E e, boolean overwritten) - throws EntityAlreadyExistsException; - - /** - * Update the entity. - * - * @param ident The identifier of the entity which need be stored. - * @param entityType The type of the entity. - * @return The entity after updating. - * @throws NoSuchEntityException If the entity is not exist. - */ - E update( - NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws NoSuchEntityException; - - /** - * Retrieves the entity associated with the identifier and the entity type. - * - * @param ident The identifier of the entity. - * @param entityType The type of the entity. - * @return The entity associated with the identifier and the entity type, or null if the key does - * not exist. - * @throws IOException If an I/O exception occurs during retrieval. - */ - E get(NameIdentifier ident, Entity.EntityType entityType) - throws IOException; - - /** - * Deletes the entity associated with the identifier and the entity type. - * - * @param ident The identifier of the entity. - * @param entityType The type of the entity. - * @param cascade True, If you need to cascade delete entities, else false. - * @return True, if the entity was successfully deleted, else false. - */ - boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade); -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java deleted file mode 100644 index 72b4aebc224..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/RelationEntityStore.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.storage.relation; - -import static com.datastrato.gravitino.Configs.ENTITY_RELATION_STORE; - -import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Configs; -import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.EntityAlreadyExistsException; -import com.datastrato.gravitino.EntitySerDe; -import com.datastrato.gravitino.EntityStore; -import com.datastrato.gravitino.HasIdentifier; -import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; -import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.storage.relation.mysql.MySQLBackend; -import com.datastrato.gravitino.utils.Executable; -import com.google.common.collect.ImmutableMap; -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Relation store to store entities. This means we can store entities in a relational store. I.e., - * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link - * RelationBackend} interface - */ -public class RelationEntityStore implements EntityStore { - private static final Logger LOGGER = LoggerFactory.getLogger(RelationEntityStore.class); - public static final ImmutableMap RELATION_BACKENDS = - ImmutableMap.of(Configs.DEFAULT_ENTITY_RELATION_STORE, MySQLBackend.class.getCanonicalName()); - private RelationBackend backend; - - @Override - public void initialize(Config config) throws RuntimeException { - this.backend = createRelationEntityBackend(config); - } - - private static RelationBackend createRelationEntityBackend(Config config) { - String backendName = config.get(ENTITY_RELATION_STORE); - String className = RELATION_BACKENDS.getOrDefault(backendName, backendName); - if (Objects.isNull(className)) { - throw new RuntimeException("Unsupported backend type..." + backendName); - } - - try { - RelationBackend relationBackend = - (RelationBackend) Class.forName(className).getDeclaredConstructor().newInstance(); - relationBackend.initialize(config); - return relationBackend; - } catch (Exception e) { - LOGGER.error("Failed to create and initialize RelationBackend by name '{}'.", backendName, e); - throw new RuntimeException( - "Failed to create and initialize RelationBackend by name: " + backendName, e); - } - } - - @Override - public void setSerDe(EntitySerDe entitySerDe) { - throw new UnsupportedOperationException("Unsupported operation in relation entity store."); - } - - @Override - public List list( - Namespace namespace, Class type, Entity.EntityType entityType) throws IOException { - return backend.list(namespace, entityType); - } - - @Override - public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws IOException { - return backend.exists(ident, entityType); - } - - @Override - public void put(E e, boolean overwritten) - throws IOException, EntityAlreadyExistsException { - backend.insert(e, overwritten); - } - - @Override - public E update( - NameIdentifier ident, Class type, Entity.EntityType entityType, Function updater) - throws IOException, NoSuchEntityException, AlreadyExistsException { - return backend.update(ident, entityType, updater); - } - - @Override - public E get( - NameIdentifier ident, Entity.EntityType entityType, Class e) - throws NoSuchEntityException, IOException { - E entity = backend.get(ident, entityType); - if (entity == null) { - throw new NoSuchEntityException(ident.toString()); - } - return entity; - } - - @Override - public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) - throws IOException { - return backend.delete(ident, entityType, cascade); - } - - @Override - public R executeInTransaction(Executable executable) { - throw new UnsupportedOperationException("Unsupported operation in relation entity store."); - } - - @Override - public void close() throws IOException { - backend.close(); - } -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java deleted file mode 100644 index 355fc2d2a81..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.storage.relation.mysql; - -import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.EntityAlreadyExistsException; -import com.datastrato.gravitino.HasIdentifier; -import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.storage.relation.RelationBackend; -import java.io.IOException; -import java.util.List; -import java.util.function.Function; - -/** - * {@link MySQLBackend} is a MySQL implementation of RelationBackend interface. If we want to use - * another relation implementation, We can just implement {@link RelationBackend} interface and use - * it in the Gravitino. - */ -public class MySQLBackend implements RelationBackend { - - /** Initialize the MySQL backend instance. */ - @Override - public void initialize(Config config) { - throw new UnsupportedOperationException("Unsupported operation now."); - } - - @Override - public List list( - Namespace namespace, Entity.EntityType entityType) throws NoSuchEntityException { - throw new UnsupportedOperationException("Unsupported operation now."); - } - - @Override - public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { - throw new UnsupportedOperationException("Unsupported operation now."); - } - - @Override - public void insert(E e, boolean overwritten) - throws EntityAlreadyExistsException { - throw new UnsupportedOperationException("Unsupported operation now."); - } - - @Override - public E update( - NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws NoSuchEntityException { - throw new UnsupportedOperationException("Unsupported operation now."); - } - - @Override - public E get( - NameIdentifier ident, Entity.EntityType entityType) throws IOException { - throw new UnsupportedOperationException("Unsupported operation now."); - } - - @Override - public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { - throw new UnsupportedOperationException("Unsupported operation now."); - } - - @Override - public void close() throws IOException { - throw new UnsupportedOperationException("Unsupported operation now."); - } -} From eb70ecf320848e16e6b2505ab54aa72e09425123 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Fri, 2 Feb 2024 15:11:36 +0800 Subject: [PATCH 05/34] add the code skeleton for mysql backend operating metalake --- core/build.gradle.kts | 2 + .../com/datastrato/gravitino/Configs.java | 5 + .../mysql/mapper/MetalakeMetaMapper.java | 85 +++++++++ .../mysql/orm/SqlSessionFactoryHelper.java | 72 +++++++ .../relation/mysql/orm/SqlSessions.java | 65 +++++++ .../storage/relation/mysql/po/MetalakePO.java | 141 ++++++++++++++ .../relation/mysql/utils/POConverters.java | 42 +++++ .../relational/mysql/MySQLBackend.java | 177 ++++++++++++++++-- .../mysql_init/mysql_backend_init.sql | 13 ++ gradle/libs.versions.toml | 2 + 10 files changed, 592 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java create mode 100644 core/src/main/resources/mysql_init/mysql_backend_init.sql diff --git a/core/build.gradle.kts b/core/build.gradle.kts index ed3cc93eef1..6691e97ccde 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { implementation(libs.bundles.prometheus) implementation(libs.caffeine) implementation(libs.commons.io) + implementation(libs.commons.dbcp2) implementation(libs.commons.lang3) implementation(libs.guava) implementation(libs.protobuf.java.util) { @@ -24,6 +25,7 @@ dependencies { .because("Brings in Guava for Android, which we don't want (and breaks multimaps).") } implementation(libs.rocksdbjni) + implementation(libs.mybatis) annotationProcessor(libs.lombok) compileOnly(libs.lombok) diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 4cc276c6b11..57b71865b3b 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -22,6 +22,11 @@ public interface Configs { String DEFAULT_ENTITY_RELATIONAL_STORE = "MySQLBackend"; String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; + String MYSQL_ENTITY_STORE_URL_KEY = "gravitino.entity.store.mysql.url"; + String MYSQL_ENTITY_STORE_DRIVER_NAME_KEY = "gravitino.entity.store.mysql.driverName"; + String MYSQL_ENTITY_STORE_USERNAME_KEY = "gravitino.entity.store.mysql.username"; + String MYSQL_ENTITY_STORE_PASSWORD_KEY = "gravitino.entity.store.mysql.password"; + String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; Long DEFAULT_KV_DELETE_AFTER_TIME = 604800000L; // 7 days diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java new file mode 100644 index 00000000000..2ab836d6578 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java @@ -0,0 +1,85 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.mapper; + +import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import java.util.List; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +public interface MetalakeMetaMapper { + String TABLE_NAME = "metalake_meta"; + + @Select( + "SELECT id, metalake_name as metalakeName, metalake_comment as metalakeComment," + + " properties, audit_info as auditInfo, schema_version as schemaVersion" + + " FROM " + + TABLE_NAME) + List listMetalakePOs(); + + @Select( + "SELECT id, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion" + + " FROM " + + TABLE_NAME + + " WHERE metalake_name = #{metalakeName}") + MetalakePO selectMetalakeMetaByName(@Param("metalakeName") String name); + + @Select("SELECT id FROM " + TABLE_NAME + " WHERE metalake_name = #{metalakeName}") + Long selectMetalakeIdMetaByName(@Param("metalakeName") String name); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" + + " VALUES(" + + " #{metalakeMeta.id}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}" + + " )") + void insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" + + " VALUES(" + + " #{metalakeMeta.id}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}") + void insertMetalakeMetaWithUpdate(@Param("metalakeMeta") MetalakePO metalakePO); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}" + + " WHERE id = #{metalakeMeta.id}") + void updateMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); + + @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") + Integer deleteMetalakeMetaById(@Param("id") Long id); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java new file mode 100644 index 00000000000..9985bc5ab4d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java @@ -0,0 +1,72 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.orm; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; +import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; +import com.google.common.base.Preconditions; +import java.time.Duration; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.pool2.impl.BaseObjectPoolConfig; +import org.apache.ibatis.mapping.Environment; +import org.apache.ibatis.session.Configuration; +import org.apache.ibatis.session.SqlSessionFactory; +import org.apache.ibatis.session.SqlSessionFactoryBuilder; +import org.apache.ibatis.transaction.TransactionFactory; +import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; + +public class SqlSessionFactoryHelper { + private static volatile SqlSessionFactory sqlSessionFactory; + private static final SqlSessionFactoryHelper INSTANCE = new SqlSessionFactoryHelper(); + + public static SqlSessionFactoryHelper getInstance() { + return INSTANCE; + } + + @SuppressWarnings("deprecation") + public void init(Config config) { + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setUrl(config.getRawString(Configs.MYSQL_ENTITY_STORE_URL_KEY)); + dataSource.setDriverClassName(config.getRawString(Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)); + dataSource.setUsername(config.getRawString(Configs.MYSQL_ENTITY_STORE_USERNAME_KEY, "")); + dataSource.setPassword(config.getRawString(Configs.MYSQL_ENTITY_STORE_PASSWORD_KEY, "")); + // close the auto commit, so that need manual commit + dataSource.setDefaultAutoCommit(false); + dataSource.setMaxWaitMillis(1000L); + dataSource.setMaxTotal(20); + dataSource.setMaxIdle(5); + dataSource.setMinIdle(0); + dataSource.setLogAbandoned(true); + dataSource.setRemoveAbandonedOnBorrow(true); + dataSource.setRemoveAbandonedTimeout(60); + dataSource.setTimeBetweenEvictionRunsMillis(Duration.ofMillis(10 * 60 * 1000L).toMillis()); + dataSource.setTestOnBorrow(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW); + dataSource.setTestWhileIdle(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE); + dataSource.setMinEvictableIdleTimeMillis(1000); + dataSource.setNumTestsPerEvictionRun(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN); + dataSource.setTestOnReturn(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN); + dataSource.setSoftMinEvictableIdleTimeMillis( + BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME.toMillis()); + dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO); + TransactionFactory transactionFactory = new JdbcTransactionFactory(); + Environment environment = new Environment("development", transactionFactory, dataSource); + Configuration configuration = new Configuration(environment); + configuration.addMapper(MetalakeMetaMapper.class); + if (sqlSessionFactory == null) { + synchronized (SqlSessionFactoryHelper.class) { + if (sqlSessionFactory == null) { + sqlSessionFactory = new SqlSessionFactoryBuilder().build(configuration); + } + } + } + } + + public SqlSessionFactory getSqlSessionFactory() { + Preconditions.checkState(sqlSessionFactory != null, "SqlSessionFactory is not initialized."); + return sqlSessionFactory; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java new file mode 100644 index 00000000000..13db581d41a --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java @@ -0,0 +1,65 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.orm; + +import java.io.Closeable; +import java.io.IOException; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.TransactionIsolationLevel; + +public final class SqlSessions implements Closeable { + private static final ThreadLocal sessions = new ThreadLocal<>(); + + private SqlSessions() {} + + public static SqlSession getSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession == null) { + sqlSession = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .openSession(TransactionIsolationLevel.READ_COMMITTED); + sessions.set(sqlSession); + return sqlSession; + } + return sqlSession; + } + + public static void commitAndCloseSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession != null) { + sqlSession.commit(); + sqlSession.close(); + sessions.remove(); + } + } + + public static void rollbackAndCloseSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession != null) { + sqlSession.rollback(); + sqlSession.close(); + sessions.remove(); + } + } + + public static void closeSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession != null) { + sqlSession.close(); + sessions.remove(); + } + } + + public static T getMapper(Class className) { + return (T) getSqlSession().getMapper(className); + } + + @Override + public void close() throws IOException { + sessions.remove(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java new file mode 100644 index 00000000000..11328b8083d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java @@ -0,0 +1,141 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.po; + +import com.datastrato.gravitino.json.JsonUtils; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Objects; +import java.util.Map; + +public class MetalakePO { + private Long id; + private String metalakeName; + private String metalakeComment; + private String properties; + private String auditInfo; + private String schemaVersion; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getMetalakeName() { + return metalakeName; + } + + public void setMetalakeName(String metalakeName) { + this.metalakeName = metalakeName; + } + + public String getMetalakeComment() { + return metalakeComment; + } + + public void setMetalakeComment(String metalakeComment) { + this.metalakeComment = metalakeComment; + } + + public String getProperties() { + return properties; + } + + public void setProperties(String properties) { + this.properties = properties; + } + + public String getAuditInfo() { + return auditInfo; + } + + public void setAuditInfo(String auditInfo) { + this.auditInfo = auditInfo; + } + + public String getSchemaVersion() { + return schemaVersion; + } + + public void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MetalakePO)) { + return false; + } + MetalakePO that = (MetalakePO) o; + return Objects.equal(getId(), that.getId()) + && Objects.equal(getMetalakeName(), that.getMetalakeName()) + && Objects.equal(getMetalakeComment(), that.getMetalakeComment()) + && Objects.equal(getProperties(), that.getProperties()) + && Objects.equal(getAuditInfo(), that.getAuditInfo()) + && Objects.equal(getSchemaVersion(), that.getSchemaVersion()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getId(), + getMetalakeName(), + getMetalakeComment(), + getProperties(), + getAuditInfo(), + getSchemaVersion()); + } + + public static class Builder { + private final MetalakePO metalakePO; + + public Builder() { + metalakePO = new MetalakePO(); + } + + public MetalakePO.Builder withId(Long id) { + metalakePO.id = id; + return this; + } + + public MetalakePO.Builder withMetalakeName(String name) { + metalakePO.metalakeName = name; + return this; + } + + public MetalakePO.Builder withMetalakeComment(String comment) { + metalakePO.metalakeComment = comment; + return this; + } + + public MetalakePO.Builder withProperties(Map properties) + throws JsonProcessingException { + metalakePO.properties = JsonUtils.objectMapper().writeValueAsString(properties); + return this; + } + + public MetalakePO.Builder withAuditInfo(AuditInfo auditInfo) throws JsonProcessingException { + metalakePO.auditInfo = JsonUtils.objectMapper().writeValueAsString(auditInfo); + return this; + } + + public MetalakePO.Builder withVersion(SchemaVersion version) throws JsonProcessingException { + metalakePO.schemaVersion = JsonUtils.objectMapper().writeValueAsString(version); + return this; + } + + public MetalakePO build() { + return metalakePO; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java new file mode 100644 index 00000000000..0c366b847fb --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java @@ -0,0 +1,42 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.utils; + +import com.datastrato.gravitino.json.JsonUtils; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.Map; + +public class POConverters { + private POConverters() {} + + public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) throws JsonProcessingException { + return new MetalakePO.Builder() + .withId(baseMetalake.id()) + .withMetalakeName(baseMetalake.name()) + .withMetalakeComment(baseMetalake.comment()) + .withProperties(baseMetalake.properties()) + .withAuditInfo((AuditInfo) baseMetalake.auditInfo()) + .withVersion(baseMetalake.getVersion()) + .build(); + } + + public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) throws JsonProcessingException { + return new BaseMetalake.Builder() + .withId(metalakePO.getId()) + .withName(metalakePO.getMetalakeName()) + .withComment(metalakePO.getMetalakeComment()) + .withProperties(JsonUtils.objectMapper().readValue(metalakePO.getProperties(), Map.class)) + .withAuditInfo( + JsonUtils.objectMapper().readValue(metalakePO.getAuditInfo(), AuditInfo.class)) + .withVersion( + JsonUtils.objectMapper().readValue(metalakePO.getSchemaVersion(), SchemaVersion.class)) + .build(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java index 06aa430a16b..e1f31acc9ac 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java @@ -10,11 +10,23 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.storage.relational.RelationalBackend; +import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessionFactoryHelper; +import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessions; +import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relation.mysql.utils.POConverters; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Preconditions; import java.io.IOException; import java.util.List; +import java.util.Objects; import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.ibatis.session.SqlSession; /** * {@link MySQLBackend} is a MySQL implementation of RelationalBackend interface. If we want to use @@ -23,45 +35,186 @@ */ public class MySQLBackend implements RelationalBackend { - private static final String UNSUPPORTED_OPERATION_MSG = "Unsupported operation now."; - /** Initialize the MySQL backend instance. */ @Override - public void initialize(Config config) {} + public void initialize(Config config) { + SqlSessionFactoryHelper.getInstance().init(config); + } @Override public List list( - Namespace namespace, Entity.EntityType entityType) throws NoSuchEntityException { - throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG); + Namespace namespace, Entity.EntityType entityType) { + try (SqlSession session = SqlSessions.getSqlSession()) { + switch (entityType) { + case METALAKE: + List metalakePOS = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .listMetalakePOs(); + return (List) + metalakePOS.stream() + .map( + metalakePO -> { + try { + return POConverters.fromMetalakePO(metalakePO); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for list operation", entityType)); + } + } finally { + SqlSessions.closeSqlSession(); + } } @Override public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { - throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG); + try (SqlSession session = SqlSessions.getSqlSession()) { + switch (entityType) { + case METALAKE: + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + return metalakePO != null; + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for exists operation", entityType)); + } + } finally { + SqlSessions.closeSqlSession(); + } } @Override public void insert(E e, boolean overwritten) throws EntityAlreadyExistsException { - throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG); + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + if (e instanceof BaseMetalake) { + MetalakePO metalakePO = POConverters.toMetalakePO((BaseMetalake) e); + if (overwritten) { + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .insertMetalakeMetaWithUpdate(metalakePO); + } else { + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .insertMetalakeMeta(metalakePO); + } + SqlSessions.commitAndCloseSqlSession(); + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for put operation", e.getClass())); + } + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } } @Override public E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws NoSuchEntityException { - throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG); + throws NoSuchEntityException, AlreadyExistsException { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + switch (entityType) { + case METALAKE: + BaseMetalake oldMetalakeEntity = + POConverters.fromMetalakePO( + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name())); + BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); + Preconditions.checkArgument( + Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), + String.format( + "The updated metalake entity id: %s is not same with the metalake entity id before: %s", + newMetalakeEntity.id(), oldMetalakeEntity.id())); + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity)); + SqlSessions.commitAndCloseSqlSession(); + return (E) newMetalakeEntity; + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for update operation", entityType)); + } + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } } @Override public E get( - NameIdentifier ident, Entity.EntityType entityType) throws IOException { - throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG); + NameIdentifier ident, Entity.EntityType entityType) + throws NoSuchEntityException, IOException { + try (SqlSession session = SqlSessions.getSqlSession()) { + switch (entityType) { + case METALAKE: + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + return (E) POConverters.fromMetalakePO(metalakePO); + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for get operation", entityType)); + } + } finally { + SqlSessions.closeSqlSession(); + } } @Override public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { - throw new UnsupportedOperationException(UNSUPPORTED_OPERATION_MSG); + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + switch (entityType) { + case METALAKE: + Long metalakeId = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeIdMetaByName(ident.name()); + if (metalakeId != null) { + // delete metalake + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .deleteMetalakeMetaById(metalakeId); + if (cascade) { + // TODO We will cascade delete the metadata of sub-resources under metalake + } + SqlSessions.commitAndCloseSqlSession(); + } + return true; + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for delete operation", entityType)); + } + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } } @Override diff --git a/core/src/main/resources/mysql_init/mysql_backend_init.sql b/core/src/main/resources/mysql_init/mysql_backend_init.sql new file mode 100644 index 00000000000..1e9c0760410 --- /dev/null +++ b/core/src/main/resources/mysql_init/mysql_backend_init.sql @@ -0,0 +1,13 @@ +CREATE DATABASE IF NOT EXISTS `gravitino_meta` DEFAULT CHARACTER SET utf8mb4; +USE `gravitino_meta`; +CREATE TABLE IF NOT EXISTS `metalake_meta` +( + `id` bigint(20) unsigned NOT NULL COMMENT 'metalake id', + `metalake_name` varchar(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` varchar(256) DEFAULT '' COMMENT 'metalake comment', + `properties` mediumtext DEFAULT NULL COMMENT 'metalake properties', + `audit_info` mediumtext NOT NULL COMMENT 'metalake audit info', + `schema_version` text NOT NULL COMMENT 'metalake schema version info', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_mn` (`metalake_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'metalake metadata'; \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b683328281f..86e1bed576f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -45,6 +45,7 @@ postgresql = "42.6.0" immutables-value = "2.10.0" selenium = "3.141.59" rauschig = "1.2.0" +mybatis = "3.5.6" protobuf-plugin = "0.9.2" spotless-plugin = '6.11.0' @@ -146,6 +147,7 @@ sun-activation = { group = "com.sun.activation", name = "javax.activation", vers selenium = { group = "org.seleniumhq.selenium", name = "selenium-java", version.ref = "selenium" } rauschig = { group = "org.rauschig", name = "jarchivelib", version.ref = "rauschig" } +mybatis = { group = "org.mybatis", name = "mybatis", version.ref = "mybatis"} [bundles] log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api"] From a46424187739db715832cf2b5edb479e028f6c3a Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Sun, 4 Feb 2024 11:09:45 +0800 Subject: [PATCH 06/34] fix update --- .../mysql/mapper/MetalakeMetaMapper.java | 21 ++++++++++++------- .../relational/mysql/MySQLBackend.java | 10 ++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java index 2ab836d6578..5a34889481c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java @@ -72,13 +72,20 @@ public interface MetalakeMetaMapper { @Update( "UPDATE " + TABLE_NAME - + " SET metalake_name = #{metalakeMeta.metalakeName}," - + " metalake_comment = #{metalakeMeta.metalakeComment}," - + " properties = #{metalakeMeta.properties}," - + " audit_info = #{metalakeMeta.auditInfo}," - + " schema_version = #{metalakeMeta.schemaVersion}" - + " WHERE id = #{metalakeMeta.id}") - void updateMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); + + " SET metalake_name = #{newMetalakeMeta.metalakeName}," + + " metalake_comment = #{newMetalakeMeta.metalakeComment}," + + " properties = #{newMetalakeMeta.properties}," + + " audit_info = #{newMetalakeMeta.auditInfo}," + + " schema_version = #{newMetalakeMeta.schemaVersion}" + + " WHERE id = #{oldMetalakeMeta.id}" + + " and metalake_name = #{oldMetalakeMeta.metalakeComment}" + + " and metalake_comment = #{oldMetalakeMeta.metalakeComment}" + + " and properties = #{oldMetalakeMeta.properties}" + + " and audit_info = #{oldMetalakeMeta.auditInfo}" + + " and schema_version = #{oldMetalakeMeta.schemaVersion}") + void updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") Integer deleteMetalakeMetaById(@Param("id") Long id); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java index e1f31acc9ac..1232b2ff57f 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java @@ -130,10 +130,10 @@ public E update( try { switch (entityType) { case METALAKE: - BaseMetalake oldMetalakeEntity = - POConverters.fromMetalakePO( - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name())); + MetalakePO oldMetalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); Preconditions.checkArgument( Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), @@ -141,7 +141,7 @@ public E update( "The updated metalake entity id: %s is not same with the metalake entity id before: %s", newMetalakeEntity.id(), oldMetalakeEntity.id())); ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity)); + .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO); SqlSessions.commitAndCloseSqlSession(); return (E) newMetalakeEntity; case CATALOG: From 3f29d1f821bc75f6152887ae07ec0c5a4ce3d3b6 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 5 Feb 2024 11:55:32 +0800 Subject: [PATCH 07/34] add unit test --- core/build.gradle.kts | 1 + .../relational/mysql/MySQLBackend.java | 153 +++++---- .../mysql/mapper/MetalakeMetaMapper.java | 6 +- .../mysql/orm/SqlSessionFactoryHelper.java | 16 +- .../mysql/orm/SqlSessions.java | 38 ++- .../mysql/po/MetalakePO.java | 19 +- .../mysql/utils/POConverters.java | 25 +- core/src/main/resources/mysql/mysql_init.sql | 11 + .../mysql_init/mysql_backend_init.sql | 13 - .../relational/TestRelationalEntityStore.java | 292 ++++++++++++++---- core/src/test/resources/h2/h2-init.sql | 10 + gradle/libs.versions.toml | 2 + 12 files changed, 423 insertions(+), 163 deletions(-) rename core/src/main/java/com/datastrato/gravitino/storage/{relation => relational}/mysql/mapper/MetalakeMetaMapper.java (94%) rename core/src/main/java/com/datastrato/gravitino/storage/{relation => relational}/mysql/orm/SqlSessionFactoryHelper.java (84%) rename core/src/main/java/com/datastrato/gravitino/storage/{relation => relational}/mysql/orm/SqlSessions.java (54%) rename core/src/main/java/com/datastrato/gravitino/storage/{relation => relational}/mysql/po/MetalakePO.java (77%) rename core/src/main/java/com/datastrato/gravitino/storage/{relation => relational}/mysql/utils/POConverters.java (60%) create mode 100644 core/src/main/resources/mysql/mysql_init.sql delete mode 100644 core/src/main/resources/mysql_init/mysql_backend_init.sql create mode 100644 core/src/test/resources/h2/h2-init.sql diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 6691e97ccde..0d576f71bf9 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -38,4 +38,5 @@ dependencies { testImplementation(libs.mockito.core) testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.h2db) } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java index 1232b2ff57f..cd6183f2135 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java @@ -14,14 +14,15 @@ import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.storage.relational.RelationalBackend; -import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; -import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessionFactoryHelper; -import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessions; -import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; -import com.datastrato.gravitino.storage.relation.mysql.utils.POConverters; +import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relational.mysql.orm.SqlSessionFactoryHelper; +import com.datastrato.gravitino.storage.relational.mysql.orm.SqlSessions; +import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.mysql.utils.POConverters; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -45,54 +46,61 @@ public void initialize(Config config) { public List list( Namespace namespace, Entity.EntityType entityType) { try (SqlSession session = SqlSessions.getSqlSession()) { - switch (entityType) { - case METALAKE: - List metalakePOS = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .listMetalakePOs(); - return (List) - metalakePOS.stream() - .map( - metalakePO -> { - try { - return POConverters.fromMetalakePO(metalakePO); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for list operation", entityType)); + try { + switch (entityType) { + case METALAKE: + List metalakePOS = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .listMetalakePOs(); + return metalakePOS != null + ? metalakePOS.stream() + .map( + metalakePO -> { + try { + return (E) POConverters.fromMetalakePO(metalakePO); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()) + : new ArrayList<>(); + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for list operation", entityType)); + } + } catch (Throwable t) { + SqlSessions.closeSqlSession(); + throw new RuntimeException(t); } - } finally { - SqlSessions.closeSqlSession(); } } @Override public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { try (SqlSession session = SqlSessions.getSqlSession()) { - switch (entityType) { - case METALAKE: - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - return metalakePO != null; - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for exists operation", entityType)); + try { + switch (entityType) { + case METALAKE: + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + return metalakePO != null; + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for exists operation", entityType)); + } + } catch (Throwable t) { + SqlSessions.closeSqlSession(); + throw new RuntimeException(t); } - } finally { - SqlSessions.closeSqlSession(); } } @@ -102,18 +110,25 @@ public void insert(E e, boolean overwritten) try (SqlSession session = SqlSessions.getSqlSession()) { try { if (e instanceof BaseMetalake) { - MetalakePO metalakePO = POConverters.toMetalakePO((BaseMetalake) e); + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(e.nameIdentifier().name()); + if (!overwritten && metalakePO != null) { + throw new EntityAlreadyExistsException( + String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); + } + if (overwritten) { ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMetaWithUpdate(metalakePO); + .insertMetalakeMetaWithUpdate(POConverters.toMetalakePO((BaseMetalake) e)); } else { ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMeta(metalakePO); + .insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); } SqlSessions.commitAndCloseSqlSession(); } else { throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for put operation", e.getClass())); + String.format("Unsupported entity type: %s for insert operation", e.getClass())); } } catch (Throwable t) { SqlSessions.rollbackAndCloseSqlSession(); @@ -138,7 +153,7 @@ public E update( Preconditions.checkArgument( Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), String.format( - "The updated metalake entity id: %s is not same with the metalake entity id before: %s", + "The updated metalake entity id: %s should same with the metalake entity id before: %s", newMetalakeEntity.id(), oldMetalakeEntity.id())); ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO); @@ -164,22 +179,28 @@ public E get( NameIdentifier ident, Entity.EntityType entityType) throws NoSuchEntityException, IOException { try (SqlSession session = SqlSessions.getSqlSession()) { - switch (entityType) { - case METALAKE: - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - return (E) POConverters.fromMetalakePO(metalakePO); - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for get operation", entityType)); + try { + switch (entityType) { + case METALAKE: + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + if (metalakePO == null) { + return null; + } + return (E) POConverters.fromMetalakePO(metalakePO); + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for get operation", entityType)); + } + } catch (Throwable t) { + SqlSessions.closeSqlSession(); + throw new RuntimeException(t); } - } finally { - SqlSessions.closeSqlSession(); } } @@ -197,7 +218,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) .deleteMetalakeMetaById(metalakeId); if (cascade) { - // TODO We will cascade delete the metadata of sub-resources under metalake + // TODO We will cascade delete the metadata of sub-resources under the metalake } SqlSessions.commitAndCloseSqlSession(); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java similarity index 94% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java index 5a34889481c..f2791a22558 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java @@ -3,9 +3,9 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation.mysql.mapper; +package com.datastrato.gravitino.storage.relational.mysql.mapper; -import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; import java.util.List; import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; @@ -78,7 +78,7 @@ public interface MetalakeMetaMapper { + " audit_info = #{newMetalakeMeta.auditInfo}," + " schema_version = #{newMetalakeMeta.schemaVersion}" + " WHERE id = #{oldMetalakeMeta.id}" - + " and metalake_name = #{oldMetalakeMeta.metalakeComment}" + + " and metalake_name = #{oldMetalakeMeta.metalakeName}" + " and metalake_comment = #{oldMetalakeMeta.metalakeComment}" + " and properties = #{oldMetalakeMeta.properties}" + " and audit_info = #{oldMetalakeMeta.auditInfo}" diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessionFactoryHelper.java similarity index 84% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessionFactoryHelper.java index 9985bc5ab4d..c584c73da0a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessionFactoryHelper.java @@ -3,11 +3,11 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation.mysql.orm; +package com.datastrato.gravitino.storage.relational.mysql.orm; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; -import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; import com.google.common.base.Preconditions; import java.time.Duration; import org.apache.commons.dbcp2.BasicDataSource; @@ -19,6 +19,11 @@ import org.apache.ibatis.transaction.TransactionFactory; import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; +/** + * SqlSessionFactoryHelper maintains the MyBatis's {@link SqlSessionFactory} object, which is used + * to create the {@link org.apache.ibatis.session.SqlSession} object. It is a singleton class and + * should be initialized only once. + */ public class SqlSessionFactoryHelper { private static volatile SqlSessionFactory sqlSessionFactory; private static final SqlSessionFactoryHelper INSTANCE = new SqlSessionFactoryHelper(); @@ -27,6 +32,13 @@ public static SqlSessionFactoryHelper getInstance() { return INSTANCE; } + private SqlSessionFactoryHelper() {} + + /** + * Initialize the SqlSessionFactory object. + * + * @param config Config object to get the MySQL connection details from the config. + */ @SuppressWarnings("deprecation") public void init(Config config) { BasicDataSource dataSource = new BasicDataSource(); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessions.java similarity index 54% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessions.java index 13db581d41a..acf9a23dd84 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessions.java @@ -3,18 +3,27 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation.mysql.orm; +package com.datastrato.gravitino.storage.relational.mysql.orm; -import java.io.Closeable; -import java.io.IOException; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.TransactionIsolationLevel; -public final class SqlSessions implements Closeable { +/** + * SqlSessions is a utility class to maintain the MyBatis's {@link SqlSession} object. It is a + * thread local class and should be used to get the {@link SqlSession} object. It also provides the + * methods to commit, rollback and close the {@link SqlSession} object. + */ +public final class SqlSessions { private static final ThreadLocal sessions = new ThreadLocal<>(); private SqlSessions() {} + /** + * Get the SqlSession object. If the SqlSession object is not present in the thread local, then + * create a new SqlSession object and set it in the thread local. + * + * @return SqlSession object from the thread local storage. + */ public static SqlSession getSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession == null) { @@ -28,6 +37,10 @@ public static SqlSession getSqlSession() { return sqlSession; } + /** + * Commit the SqlSession object and close it. It also removes the SqlSession object from the + * thread local storage. + */ public static void commitAndCloseSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { @@ -37,6 +50,10 @@ public static void commitAndCloseSqlSession() { } } + /** + * Rollback the SqlSession object and close it. It also removes the SqlSession object from the + * thread local storage. + */ public static void rollbackAndCloseSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { @@ -46,6 +63,7 @@ public static void rollbackAndCloseSqlSession() { } } + /** Close the SqlSession object and remove it from the thread local storage. */ public static void closeSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { @@ -54,12 +72,14 @@ public static void closeSqlSession() { } } + /** + * Get the Mapper object from the SqlSession object. + * + * @param className the class name of the Mapper object. + * @return the Mapper object. + * @param the type of the Mapper object. + */ public static T getMapper(Class className) { return (T) getSqlSession().getMapper(className); } - - @Override - public void close() throws IOException { - sessions.remove(); - } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java similarity index 77% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java index 11328b8083d..ea515c8a3df 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java @@ -3,14 +3,10 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation.mysql.po; +package com.datastrato.gravitino.storage.relational.mysql.po; -import com.datastrato.gravitino.json.JsonUtils; -import com.datastrato.gravitino.meta.AuditInfo; -import com.datastrato.gravitino.meta.SchemaVersion; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Objects; -import java.util.Map; public class MetalakePO { private Long id; @@ -118,19 +114,18 @@ public MetalakePO.Builder withMetalakeComment(String comment) { return this; } - public MetalakePO.Builder withProperties(Map properties) - throws JsonProcessingException { - metalakePO.properties = JsonUtils.objectMapper().writeValueAsString(properties); + public MetalakePO.Builder withProperties(String properties) throws JsonProcessingException { + metalakePO.properties = properties; return this; } - public MetalakePO.Builder withAuditInfo(AuditInfo auditInfo) throws JsonProcessingException { - metalakePO.auditInfo = JsonUtils.objectMapper().writeValueAsString(auditInfo); + public MetalakePO.Builder withAuditInfo(String auditInfo) throws JsonProcessingException { + metalakePO.auditInfo = auditInfo; return this; } - public MetalakePO.Builder withVersion(SchemaVersion version) throws JsonProcessingException { - metalakePO.schemaVersion = JsonUtils.objectMapper().writeValueAsString(version); + public MetalakePO.Builder withVersion(String version) throws JsonProcessingException { + metalakePO.schemaVersion = version; return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java similarity index 60% rename from core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java index 0c366b847fb..405651aecf2 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java @@ -3,30 +3,45 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relation.mysql.utils; +package com.datastrato.gravitino.storage.relational.mysql.utils; import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; import java.util.Map; +/** POConverters is a utility class to convert PO to Base and vice versa. */ public class POConverters { private POConverters() {} + /** + * Convert {@link BaseMetalake} to {@link MetalakePO} + * + * @param baseMetalake BaseMetalake object + * @return MetalakePO object from BaseMetalake object + * @throws JsonProcessingException + */ public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) throws JsonProcessingException { return new MetalakePO.Builder() .withId(baseMetalake.id()) .withMetalakeName(baseMetalake.name()) .withMetalakeComment(baseMetalake.comment()) - .withProperties(baseMetalake.properties()) - .withAuditInfo((AuditInfo) baseMetalake.auditInfo()) - .withVersion(baseMetalake.getVersion()) + .withProperties(JsonUtils.objectMapper().writeValueAsString(baseMetalake.properties())) + .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(baseMetalake.auditInfo())) + .withVersion(JsonUtils.objectMapper().writeValueAsString(baseMetalake.getVersion())) .build(); } + /** + * Convert {@link MetalakePO} to {@link BaseMetalake} + * + * @param metalakePO MetalakePO object + * @return BaseMetalake object from MetalakePO object + * @throws JsonProcessingException + */ public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) throws JsonProcessingException { return new BaseMetalake.Builder() .withId(metalakePO.getId()) diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql new file mode 100644 index 00000000000..54b28a6d7c8 --- /dev/null +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS `metalake_meta` +( + `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', + `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_mn` (`metalake_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'metalake metadata'; \ No newline at end of file diff --git a/core/src/main/resources/mysql_init/mysql_backend_init.sql b/core/src/main/resources/mysql_init/mysql_backend_init.sql deleted file mode 100644 index 1e9c0760410..00000000000 --- a/core/src/main/resources/mysql_init/mysql_backend_init.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `gravitino_meta` DEFAULT CHARACTER SET utf8mb4; -USE `gravitino_meta`; -CREATE TABLE IF NOT EXISTS `metalake_meta` -( - `id` bigint(20) unsigned NOT NULL COMMENT 'metalake id', - `metalake_name` varchar(128) NOT NULL COMMENT 'metalake name', - `metalake_comment` varchar(256) DEFAULT '' COMMENT 'metalake comment', - `properties` mediumtext DEFAULT NULL COMMENT 'metalake properties', - `audit_info` mediumtext NOT NULL COMMENT 'metalake audit info', - `schema_version` text NOT NULL COMMENT 'metalake schema version info', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_mn` (`metalake_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'metalake metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index 039e3bf0f3a..d52acb0c382 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -6,99 +6,285 @@ package com.datastrato.gravitino.storage.relational; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.EntityStoreFactory; -import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relational.mysql.orm.SqlSessionFactoryHelper; +import java.io.BufferedReader; +import java.io.File; import java.io.IOException; -import java.util.function.Function; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Mockito; public class TestRelationalEntityStore { - private static EntityStore store; + private static final String MYSQL_STORE_PATH = + "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; + private static EntityStore entityStore = null; @BeforeAll public static void setUp() { + File dir = new File(DB_DIR); + if (dir.exists() || !dir.isDirectory()) { + dir.delete(); + } + dir.mkdirs(); + + // Use H2 DATABASE to simulate MySQL Config config = Mockito.mock(Config.class); - Mockito.when(config.get(Configs.ENTITY_STORE)).thenReturn(Configs.RELATIONAL_ENTITY_STORE); - Mockito.when(config.get(Configs.ENTITY_RELATIONAL_STORE)) - .thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - store = EntityStoreFactory.createEntityStore(config); - store.initialize(config); - Assertions.assertTrue(store instanceof RelationalEntityStore); + Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + .thenReturn("org.h2.Driver"); + entityStore = EntityStoreFactory.createEntityStore(config); + entityStore.initialize(config); + + // Read the ddl sql to create table + String scriptPath = "h2/h2-init.sql"; + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement()) { + URL scriptUrl = ClassLoader.getSystemResource(scriptPath); + if (scriptUrl == null) { + throw new IllegalStateException("Cannot find init sql script:" + scriptPath); + } + StringBuilder ddlBuilder = new StringBuilder(); + try (InputStreamReader inputStreamReader = + new InputStreamReader( + Files.newInputStream(Paths.get(scriptUrl.getPath())), StandardCharsets.UTF_8); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { + String line; + while ((line = bufferedReader.readLine()) != null) { + ddlBuilder.append(line).append("\n"); + } + } + statement.execute(ddlBuilder.toString()); + } catch (Exception e) { + throw new IllegalStateException("Create tables failed", e); + } + } + + @AfterEach + public void destroy() { + truncateAllTables(); } @AfterAll - public static void teardown() throws IOException { - store.close(); - store = null; + public static void tearDown() throws IOException { + dropAllTables(); + entityStore.close(); + File dir = new File(DB_DIR); + if (dir.exists()) { + dir.delete(); + } } @Test - public void testSetSerDe() { - Assertions.assertThrows(UnsupportedOperationException.class, () -> store.setSerDe(null)); + public void testPutAndGet() throws IOException { + BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); + entityStore.put(metalake, false); + BaseMetalake insertedMetalake = + entityStore.get(metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); + assertNotNull(insertedMetalake); + assertTrue(checkMetalakeEquals(metalake, insertedMetalake)); + + // overwrite false + BaseMetalake duplicateMetalake = createMetalake(1L, "test_metalake", "this is test"); + assertThrows(RuntimeException.class, () -> entityStore.put(duplicateMetalake, false)); + + // overwrite true + BaseMetalake overittenMetalake = createMetalake(1L, "test_metalake2", "this is test2"); + entityStore.put(overittenMetalake, true); + BaseMetalake insertedMetalake1 = + entityStore.get( + overittenMetalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); + assertEquals( + 1, + entityStore.list(Namespace.empty(), BaseMetalake.class, Entity.EntityType.METALAKE).size()); + assertEquals("test_metalake2", insertedMetalake1.name()); + assertEquals("this is test2", insertedMetalake1.comment()); } @Test - public void testExecuteInTransaction() { - Assertions.assertThrows( - UnsupportedOperationException.class, () -> store.executeInTransaction(null)); + public void testPutAndList() throws IOException { + BaseMetalake metalake1 = createMetalake(1L, "test_metalake1", "this is test 1"); + BaseMetalake metalake2 = createMetalake(2L, "test_metalake2", "this is test 2"); + entityStore.put(metalake1, false); + entityStore.put(metalake2, false); + List metalakes = + entityStore.list(metalake1.namespace(), BaseMetalake.class, Entity.EntityType.METALAKE); + assertNotNull(metalakes); + assertEquals(2, metalakes.size()); + assertTrue(checkMetalakeEquals(metalake1, metalakes.get(0))); + assertTrue(checkMetalakeEquals(metalake2, metalakes.get(1))); } @Test - public void testExists() throws IOException { - NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> store.exists(nameIdentifier, Entity.EntityType.METALAKE)); + public void testPutAndDelete() throws IOException { + BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); + entityStore.put(metalake, false); + entityStore.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, false); + assertThrows( + NoSuchEntityException.class, + () -> + entityStore.get( + metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class)); } @Test - public void testPut() throws IOException { - BaseMetalake metalake = Mockito.mock(BaseMetalake.class); - Assertions.assertThrows(UnsupportedOperationException.class, () -> store.put(metalake, false)); + public void testPutAndUpdate() throws IOException { + BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); + entityStore.put(metalake, false); + + assertThrows( + RuntimeException.class, + () -> + entityStore.update( + metalake.nameIdentifier(), + BaseMetalake.class, + Entity.EntityType.METALAKE, + m -> { + BaseMetalake.Builder builder = + new BaseMetalake.Builder() + // Change the id, which is not allowed + .withId(2L) + .withName("test_metalake2") + .withComment("this is test 2") + .withProperties(new HashMap<>()) + .withAuditInfo((AuditInfo) m.auditInfo()) + .withVersion(m.getVersion()); + return builder.build(); + })); + + AuditInfo changedAuditInfo = + AuditInfo.builder().withCreator("changed_creator").withCreateTime(Instant.now()).build(); + BaseMetalake updatedMetalake = + entityStore.update( + metalake.nameIdentifier(), + BaseMetalake.class, + Entity.EntityType.METALAKE, + m -> { + BaseMetalake.Builder builder = + new BaseMetalake.Builder() + .withId(m.id()) + .withName("test_metalake2") + .withComment("this is test 2") + .withProperties(new HashMap<>()) + .withAuditInfo(changedAuditInfo) + .withVersion(m.getVersion()); + return builder.build(); + }); + BaseMetalake storedMetalake = + entityStore.get( + updatedMetalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); + assertEquals(metalake.id(), storedMetalake.id()); + assertEquals("test_metalake2", updatedMetalake.name()); + assertEquals("this is test 2", updatedMetalake.comment()); + assertEquals(changedAuditInfo.creator(), updatedMetalake.auditInfo().creator()); } - @Test - public void testGet() { - NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> store.get(nameIdentifier, Entity.EntityType.METALAKE, BaseMetalake.class)); + private static BaseMetalake createMetalake(Long id, String name, String comment) { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + return new BaseMetalake.Builder() + .withId(id) + .withName(name) + .withComment(comment) + .withProperties(new HashMap<>()) + .withAuditInfo(auditInfo) + .withVersion(SchemaVersion.V_0_1) + .build(); } - @Test - public void testUpdate() { - NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); - Function function = Mockito.mock(Function.class); - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> - store.update(nameIdentifier, BaseMetalake.class, Entity.EntityType.METALAKE, function)); + private static boolean checkMetalakeEquals(BaseMetalake expected, BaseMetalake actual) { + return expected.id().equals(actual.id()) + && expected.name().equals(actual.name()) + && expected.comment().equals(actual.comment()) + && expected.properties().equals(actual.properties()) + && expected.auditInfo().equals(actual.auditInfo()) + && expected.getVersion().equals(actual.getVersion()); } - @Test - public void testList() { - Namespace namespace = Mockito.mock(Namespace.class); - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> store.list(namespace, BaseMetalake.class, Entity.EntityType.METALAKE)); + private static void truncateAllTables() { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + try (Statement statement = connection.createStatement()) { + String query = "SHOW TABLES"; + List tableList = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(query)) { + while (rs.next()) { + tableList.add(rs.getString(1)); + } + } + for (String table : tableList) { + statement.execute("TRUNCATE TABLE " + table); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Clear table failed", e); + } } - @Test - public void testDelete() { - NameIdentifier nameIdentifier = Mockito.mock(NameIdentifier.class); - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> store.delete(nameIdentifier, Entity.EntityType.METALAKE, false)); + private static void dropAllTables() { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + try (Statement statement = connection.createStatement()) { + String query = "SHOW TABLES"; + List tableList = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(query)) { + while (rs.next()) { + tableList.add(rs.getString(1)); + } + } + for (String table : tableList) { + statement.execute("DROP TABLE " + table); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Drop table failed", e); + } } } diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql new file mode 100644 index 00000000000..7f5590ac8e6 --- /dev/null +++ b/core/src/test/resources/h2/h2-init.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS `metalake_meta` ( + `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', + `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', + PRIMARY KEY (id), + CONSTRAINT uk_mn UNIQUE (metalake_name) +) ENGINE = InnoDB; \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 86e1bed576f..24f8bf0dee8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -46,6 +46,7 @@ immutables-value = "2.10.0" selenium = "3.141.59" rauschig = "1.2.0" mybatis = "3.5.6" +h2db = "1.4.200" protobuf-plugin = "0.9.2" spotless-plugin = '6.11.0' @@ -148,6 +149,7 @@ sun-activation = { group = "com.sun.activation", name = "javax.activation", vers selenium = { group = "org.seleniumhq.selenium", name = "selenium-java", version.ref = "selenium" } rauschig = { group = "org.rauschig", name = "jarchivelib", version.ref = "rauschig" } mybatis = { group = "org.mybatis", name = "mybatis", version.ref = "mybatis"} +h2db = { group = "com.h2database", name = "h2", version.ref = "h2db"} [bundles] log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api"] From 5cc6d402f7aa193ff4225d74201ad220537af78a Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 6 Feb 2024 14:16:29 +0800 Subject: [PATCH 08/34] fix comments --- LICENSE.bin | 1 + .../relational/mysql/MySQLBackend.java | 4 +- .../SqlSessionFactoryHelper.java | 8 +- .../mysql/{orm => session}/SqlSessions.java | 8 +- .../relational/TestRelationalEntityStore.java | 2 +- .../session/TestSqlSessionFactoryHelper.java | 102 ++++++++++++++++++ .../mysql/session/TestSqlSessions.java | 83 ++++++++++++++ 7 files changed, 203 insertions(+), 5 deletions(-) rename core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/{orm => session}/SqlSessionFactoryHelper.java (93%) rename core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/{orm => session}/SqlSessions.java (92%) create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java diff --git a/LICENSE.bin b/LICENSE.bin index 806a7d64b2f..3456cec1422 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -329,6 +329,7 @@ J2ObjC SQLite JDBC Driver Immutables + MyBatis This product bundles various third-party components also under the Apache Software Foundation License 1.1 diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java index cd6183f2135..2feb8ddfb8e 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java @@ -15,9 +15,9 @@ import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.storage.relational.RelationalBackend; import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; -import com.datastrato.gravitino.storage.relational.mysql.orm.SqlSessionFactoryHelper; -import com.datastrato.gravitino.storage.relational.mysql.orm.SqlSessions; import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessionFactoryHelper; +import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessions; import com.datastrato.gravitino.storage.relational.mysql.utils.POConverters; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java similarity index 93% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessionFactoryHelper.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java index c584c73da0a..56247318eba 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java @@ -3,11 +3,12 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.orm; +package com.datastrato.gravitino.storage.relational.mysql.session; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.time.Duration; import org.apache.commons.dbcp2.BasicDataSource; @@ -34,6 +35,11 @@ public static SqlSessionFactoryHelper getInstance() { private SqlSessionFactoryHelper() {} + @VisibleForTesting + static void setSqlSessionFactory(SqlSessionFactory sessionFactory) { + sqlSessionFactory = sessionFactory; + } + /** * Initialize the SqlSessionFactory object. * diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java similarity index 92% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessions.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java index acf9a23dd84..9b5311e7b8d 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/orm/SqlSessions.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java @@ -3,8 +3,9 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.orm; +package com.datastrato.gravitino.storage.relational.mysql.session; +import com.google.common.annotations.VisibleForTesting; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.TransactionIsolationLevel; @@ -18,6 +19,11 @@ public final class SqlSessions { private SqlSessions() {} + @VisibleForTesting + static ThreadLocal getSessions() { + return sessions; + } + /** * Get the SqlSession object. If the SqlSession object is not present in the thread local, then * create a new SqlSession object and set it in the thread local. diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index d52acb0c382..a3dbc337ff3 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -26,7 +26,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relational.mysql.orm.SqlSessionFactoryHelper; +import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessionFactoryHelper; import java.io.BufferedReader; import java.io.File; import java.io.IOException; diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java new file mode 100644 index 00000000000..1ebeac5c2f0 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.session; + +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.datastrato.gravitino.Config; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.UUID; +import org.apache.commons.dbcp2.BasicDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestSqlSessionFactoryHelper { + private static final String MYSQL_STORE_PATH = + "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; + + private static Config config; + + @BeforeAll + public static void setUp() { + File dir = new File(DB_DIR); + if (dir.exists() || !dir.isDirectory()) { + dir.delete(); + } + dir.mkdirs(); + + config = Mockito.mock(Config.class); + Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + .thenReturn("org.h2.Driver"); + } + + @BeforeEach + public void init() { + SqlSessionFactoryHelper.setSqlSessionFactory(null); + } + + @AfterEach + public void cleanUp() { + SqlSessionFactoryHelper.setSqlSessionFactory(null); + } + + @AfterAll + public static void tearDown() throws IOException { + File dir = new File(DB_DIR); + if (dir.exists()) { + dir.delete(); + } + } + + @Test + public void testGetInstance() { + SqlSessionFactoryHelper instance = SqlSessionFactoryHelper.getInstance(); + assertNotNull(instance); + } + + @Test + public void testInit() throws SQLException { + SqlSessionFactoryHelper.getInstance().init(config); + assertNotNull(SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); + BasicDataSource dataSource = + (BasicDataSource) + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getEnvironment() + .getDataSource(); + assertEquals("org.h2.Driver", dataSource.getDriverClassName()); + assertEquals(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY), dataSource.getUrl()); + } + + @Test + public void testGetSqlSessionFactoryWithoutInit() { + assertThrows( + IllegalStateException.class, + () -> SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java new file mode 100644 index 00000000000..57c4a39e606 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java @@ -0,0 +1,83 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.session; + +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.datastrato.gravitino.Config; +import java.io.File; +import java.io.IOException; +import java.util.UUID; +import org.apache.ibatis.session.SqlSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestSqlSessions { + private static final String MYSQL_STORE_PATH = + "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; + + @BeforeAll + public static void setUp() { + File dir = new File(DB_DIR); + if (dir.exists() || !dir.isDirectory()) { + dir.delete(); + } + dir.mkdirs(); + + Config config = Mockito.mock(Config.class); + Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + .thenReturn("org.h2.Driver"); + SqlSessionFactoryHelper.getInstance().init(config); + } + + @AfterAll + public static void tearDown() throws IOException { + File dir = new File(DB_DIR); + if (dir.exists()) { + dir.delete(); + } + } + + @Test + public void testOpenAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.closeSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } + + @Test + public void testOpenAndCommitAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.commitAndCloseSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } + + @Test + public void testOpenAndRollbackAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.rollbackAndCloseSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } +} From 4a152c830044952bb3383693e4c4e4599d667dd7 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 6 Feb 2024 15:16:30 +0800 Subject: [PATCH 09/34] fix comments and uts --- .../com/datastrato/gravitino/Configs.java | 30 ++++++ .../relational/mysql/MySQLBackend.java | 9 +- .../relational/mysql/po/MetalakePO.java | 7 +- .../session/SqlSessionFactoryHelper.java | 8 +- .../relational/mysql/session/SqlSessions.java | 2 +- core/src/main/resources/mysql/mysql_init.sql | 5 + .../relational/TestRelationalEntityStore.java | 12 +-- ...FactoryHelper.java => TestSqlSession.java} | 47 ++++++++-- .../mysql/session/TestSqlSessions.java | 83 ----------------- .../mysql/utils/TestPOConverters.java | 93 +++++++++++++++++++ core/src/test/resources/h2/h2-init.sql | 5 + 11 files changed, 191 insertions(+), 110 deletions(-) rename core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/{TestSqlSessionFactoryHelper.java => TestSqlSession.java} (65%) delete mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 57b71865b3b..1243a219b11 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -64,6 +64,36 @@ public interface Configs { .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .createWithDefault(DEFAULT_ENTITY_RELATIONAL_STORE); + ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_URL = + new ConfigBuilder(MYSQL_ENTITY_STORE_URL_KEY) + .doc("Connection URL of `MySQLBackend`") + .version("0.5.0") + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME = + new ConfigBuilder(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY) + .doc("Driver Name of `MySQLBackend`") + .version("0.5.0") + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); + + ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME = + new ConfigBuilder(MYSQL_ENTITY_STORE_USERNAME_KEY) + .doc("Username of `MySQLBackend`") + .version("0.5.0") + .stringConf() + .createWithDefault(""); + + ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_PASSWORD = + new ConfigBuilder(MYSQL_ENTITY_STORE_PASSWORD_KEY) + .doc("Password of `MySQLBackend`") + .version("0.5.0") + .stringConf() + .createWithDefault(""); + ConfigEntry ENTRY_KV_ROCKSDB_BACKEND_PATH = new ConfigBuilder(ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY) .doc( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java index 2feb8ddfb8e..5816dc0d288 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java @@ -73,8 +73,9 @@ public List list( String.format("Unsupported entity type: %s for list operation", entityType)); } } catch (Throwable t) { - SqlSessions.closeSqlSession(); throw new RuntimeException(t); + } finally { + SqlSessions.closeSqlSession(); } } } @@ -98,8 +99,9 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { String.format("Unsupported entity type: %s for exists operation", entityType)); } } catch (Throwable t) { - SqlSessions.closeSqlSession(); throw new RuntimeException(t); + } finally { + SqlSessions.closeSqlSession(); } } } @@ -198,8 +200,9 @@ public E get( String.format("Unsupported entity type: %s for get operation", entityType)); } } catch (Throwable t) { - SqlSessions.closeSqlSession(); throw new RuntimeException(t); + } finally { + SqlSessions.closeSqlSession(); } } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java index ea515c8a3df..8b36eeace56 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java @@ -5,7 +5,6 @@ package com.datastrato.gravitino.storage.relational.mysql.po; -import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Objects; public class MetalakePO { @@ -114,17 +113,17 @@ public MetalakePO.Builder withMetalakeComment(String comment) { return this; } - public MetalakePO.Builder withProperties(String properties) throws JsonProcessingException { + public MetalakePO.Builder withProperties(String properties) { metalakePO.properties = properties; return this; } - public MetalakePO.Builder withAuditInfo(String auditInfo) throws JsonProcessingException { + public MetalakePO.Builder withAuditInfo(String auditInfo) { metalakePO.auditInfo = auditInfo; return this; } - public MetalakePO.Builder withVersion(String version) throws JsonProcessingException { + public MetalakePO.Builder withVersion(String version) { metalakePO.schemaVersion = version; return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java index 56247318eba..d372cd045b7 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java @@ -48,10 +48,10 @@ static void setSqlSessionFactory(SqlSessionFactory sessionFactory) { @SuppressWarnings("deprecation") public void init(Config config) { BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(config.getRawString(Configs.MYSQL_ENTITY_STORE_URL_KEY)); - dataSource.setDriverClassName(config.getRawString(Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)); - dataSource.setUsername(config.getRawString(Configs.MYSQL_ENTITY_STORE_USERNAME_KEY, "")); - dataSource.setPassword(config.getRawString(Configs.MYSQL_ENTITY_STORE_PASSWORD_KEY, "")); + dataSource.setUrl(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_URL)); + dataSource.setDriverClassName(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME)); + dataSource.setUsername(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME)); + dataSource.setPassword(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_PASSWORD)); // close the auto commit, so that need manual commit dataSource.setDefaultAutoCommit(false); dataSource.setMaxWaitMillis(1000L); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java index 9b5311e7b8d..044ed72c093 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java @@ -85,7 +85,7 @@ public static void closeSqlSession() { * @return the Mapper object. * @param the type of the Mapper object. */ - public static T getMapper(Class className) { + public static T getMapper(Class className) { return (T) getSqlSession().getMapper(className); } } diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql index 54b28a6d7c8..610b812a2b9 100644 --- a/core/src/main/resources/mysql/mysql_init.sql +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -1,3 +1,8 @@ +-- +-- Copyright 2024 Datastrato Pvt Ltd. +-- This software is licensed under the Apache License version 2. +-- + CREATE TABLE IF NOT EXISTS `metalake_meta` ( `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index a3dbc337ff3..ecbe0fb46ad 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -8,9 +8,9 @@ import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME; +import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -69,10 +69,10 @@ public static void setUp() { Config config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME)).thenReturn("sa"); + Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME)) .thenReturn("org.h2.Driver"); entityStore = EntityStoreFactory.createEntityStore(config); entityStore.initialize(config); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java similarity index 65% rename from core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java rename to core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java index 1ebeac5c2f0..299dd65a6f7 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java @@ -8,12 +8,13 @@ import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME; +import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import com.datastrato.gravitino.Config; @@ -22,6 +23,7 @@ import java.sql.SQLException; import java.util.UUID; import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -29,7 +31,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -public class TestSqlSessionFactoryHelper { +public class TestSqlSession { private static final String MYSQL_STORE_PATH = "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; @@ -47,16 +49,16 @@ public static void setUp() { config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME)).thenReturn("sa"); + Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME)) .thenReturn("org.h2.Driver"); } @BeforeEach public void init() { - SqlSessionFactoryHelper.setSqlSessionFactory(null); + SqlSessionFactoryHelper.getInstance().init(config); } @AfterEach @@ -70,6 +72,7 @@ public static void tearDown() throws IOException { if (dir.exists()) { dir.delete(); } + SqlSessionFactoryHelper.setSqlSessionFactory(null); } @Test @@ -80,6 +83,7 @@ public void testGetInstance() { @Test public void testInit() throws SQLException { + SqlSessionFactoryHelper.setSqlSessionFactory(null); SqlSessionFactoryHelper.getInstance().init(config); assertNotNull(SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); BasicDataSource dataSource = @@ -90,13 +94,38 @@ public void testInit() throws SQLException { .getEnvironment() .getDataSource(); assertEquals("org.h2.Driver", dataSource.getDriverClassName()); - assertEquals(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY), dataSource.getUrl()); + assertEquals(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_URL), dataSource.getUrl()); } @Test public void testGetSqlSessionFactoryWithoutInit() { + SqlSessionFactoryHelper.setSqlSessionFactory(null); assertThrows( IllegalStateException.class, () -> SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); } + + @Test + public void testOpenAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.closeSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } + + @Test + public void testOpenAndCommitAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.commitAndCloseSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } + + @Test + public void testOpenAndRollbackAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.rollbackAndCloseSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java deleted file mode 100644 index 57c4a39e606..00000000000 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relational.mysql.session; - -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; -import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -import com.datastrato.gravitino.Config; -import java.io.File; -import java.io.IOException; -import java.util.UUID; -import org.apache.ibatis.session.SqlSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class TestSqlSessions { - private static final String MYSQL_STORE_PATH = - "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); - private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; - - @BeforeAll - public static void setUp() { - File dir = new File(DB_DIR); - if (dir.exists() || !dir.isDirectory()) { - dir.delete(); - } - dir.mkdirs(); - - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) - .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) - .thenReturn("org.h2.Driver"); - SqlSessionFactoryHelper.getInstance().init(config); - } - - @AfterAll - public static void tearDown() throws IOException { - File dir = new File(DB_DIR); - if (dir.exists()) { - dir.delete(); - } - } - - @Test - public void testOpenAndCloseSqlSession() { - SqlSession session = SqlSessions.getSqlSession(); - assertNotNull(session); - SqlSessions.closeSqlSession(); - assertNull(SqlSessions.getSessions().get()); - } - - @Test - public void testOpenAndCommitAndCloseSqlSession() { - SqlSession session = SqlSessions.getSqlSession(); - assertNotNull(session); - SqlSessions.commitAndCloseSqlSession(); - assertNull(SqlSessions.getSessions().get()); - } - - @Test - public void testOpenAndRollbackAndCloseSqlSession() { - SqlSession session = SqlSessions.getSqlSession(); - assertNotNull(session); - SqlSessions.rollbackAndCloseSqlSession(); - assertNull(SqlSessions.getSessions().get()); - } -} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java new file mode 100644 index 00000000000..3612f099466 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java @@ -0,0 +1,93 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.datastrato.gravitino.json.JsonUtils; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class TestPOConverters { + private static final LocalDateTime FIX_DATE_TIME = LocalDateTime.of(2024, 2, 6, 0, 0, 0); + + private static final Instant FIX_INSTANT = FIX_DATE_TIME.toInstant(ZoneOffset.UTC); + + @Test + public void testFromMetalakePO() throws JsonProcessingException { + MetalakePO metalakePO = createMetalakePO(1L, "test", "this is test"); + + BaseMetalake expectedMetalake = createMetalake(1L, "test", "this is test"); + + BaseMetalake convertedMetalake = POConverters.fromMetalakePO(metalakePO); + + // Assert + assertEquals(expectedMetalake.id(), convertedMetalake.id()); + assertEquals(expectedMetalake.name(), convertedMetalake.name()); + assertEquals(expectedMetalake.comment(), convertedMetalake.comment()); + assertEquals( + expectedMetalake.properties().get("key"), convertedMetalake.properties().get("key")); + assertEquals(expectedMetalake.auditInfo().creator(), convertedMetalake.auditInfo().creator()); + assertEquals(expectedMetalake.getVersion(), convertedMetalake.getVersion()); + } + + @Test + public void testToMetalakePO() throws JsonProcessingException { + BaseMetalake metalake = createMetalake(1L, "test", "this is test"); + + MetalakePO expectedMetalakePO = createMetalakePO(1L, "test", "this is test"); + + MetalakePO actualMetalakePO = POConverters.toMetalakePO(metalake); + + // Assert + assertEquals(expectedMetalakePO.getId(), actualMetalakePO.getId()); + assertEquals(expectedMetalakePO.getMetalakeName(), actualMetalakePO.getMetalakeName()); + assertEquals(expectedMetalakePO.getMetalakeComment(), actualMetalakePO.getMetalakeComment()); + assertEquals(expectedMetalakePO.getProperties(), actualMetalakePO.getProperties()); + assertEquals(expectedMetalakePO.getAuditInfo(), actualMetalakePO.getAuditInfo()); + assertEquals(expectedMetalakePO.getSchemaVersion(), actualMetalakePO.getSchemaVersion()); + } + + private static BaseMetalake createMetalake(Long id, String name, String comment) { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); + Map properties = new HashMap<>(); + properties.put("key", "value"); + return new BaseMetalake.Builder() + .withId(id) + .withName(name) + .withComment(comment) + .withProperties(properties) + .withAuditInfo(auditInfo) + .withVersion(SchemaVersion.V_0_1) + .build(); + } + + private static MetalakePO createMetalakePO(Long id, String name, String comment) + throws JsonProcessingException { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); + Map properties = new HashMap<>(); + properties.put("key", "value"); + return new MetalakePO.Builder() + .withId(id) + .withMetalakeName(name) + .withMetalakeComment(comment) + .withProperties(JsonUtils.objectMapper().writeValueAsString(properties)) + .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(auditInfo)) + .withVersion(JsonUtils.objectMapper().writeValueAsString(SchemaVersion.V_0_1)) + .build(); + } +} diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index 7f5590ac8e6..a481db94ba0 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -1,3 +1,8 @@ +-- +-- Copyright 2024 Datastrato Pvt Ltd. +-- This software is licensed under the Apache License version 2. +-- + CREATE TABLE IF NOT EXISTS `metalake_meta` ( `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', From 42ce5ca3dace1b17c0f26acdeb5d3cd4949a59fc Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 6 Feb 2024 19:18:06 +0800 Subject: [PATCH 10/34] fix some comments --- .../datastrato/gravitino/json/JsonUtils.java | 29 ++ conf/gravitino.conf.template | 3 +- core/build.gradle.kts | 2 +- .../com/datastrato/gravitino/Configs.java | 32 ++- .../storage/relational/RelationalBackend.java | 3 +- .../relational/RelationalEntityStore.java | 6 +- .../relational/mysql/MySQLBackend.java | 261 +++++++----------- .../mysql/mapper/MetalakeMetaMapper.java | 10 +- .../session/SqlSessionFactoryHelper.java | 30 +- .../relational/mysql/utils/POConverters.java | 65 +++-- .../relational/mysql/utils/SessionUtils.java | 78 ++++++ core/src/main/resources/mysql/mysql_init.sql | 5 +- .../relational/TestRelationalEntityStore.java | 18 +- .../mysql/session/TestSqlSession.java | 15 +- .../mysql/utils/TestPOConverters.java | 6 +- core/src/test/resources/h2/h2-init.sql | 14 +- docs/gravitino-server-config.md | 5 + 17 files changed, 340 insertions(+), 242 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java diff --git a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java index 8a8f9752bf3..0c2fb6b75a7 100644 --- a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java @@ -39,6 +39,8 @@ import com.datastrato.gravitino.rel.indexes.Index; import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JacksonException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; @@ -240,6 +242,33 @@ public static ObjectMapper objectMapper() { return ObjectMapperHolder.INSTANCE; } + /** + * AnyFieldMapperHolder is a static inner class that holds the instance of ObjectMapper which can + * access any field of the object. This class utilizes the Initialization-on-demand holder idiom, + * which is a lazy-loaded singleton. This idiom takes advantage of the fact that inner classes are + * not loaded until they are referenced. It's a thread-safe and efficient way to implement a + * singleton as the instance is created when it's needed at the first time. + */ + private static class AnyFieldMapperHolder { + private static final ObjectMapper INSTANCE = + JsonMapper.builder() + .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + .configure(EnumFeature.WRITE_ENUMS_TO_LOWERCASE, true) + .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) + .build() + .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) + .registerModule(new JavaTimeModule()); + } + + /** + * Get the shared AnyFieldMapper instance for JSON serialization/deserialization. + * + * @return The ObjectMapper instance. + */ + public static ObjectMapper anyFieldMapper() { + return AnyFieldMapperHolder.INSTANCE; + } + /** * Get a list of strings from a JSON node property. * diff --git a/conf/gravitino.conf.template b/conf/gravitino.conf.template index 3cd1192b696..5a6cf006380 100644 --- a/conf/gravitino.conf.template +++ b/conf/gravitino.conf.template @@ -27,8 +27,9 @@ gravitino.server.webserver.requestHeaderSize = 131072 gravitino.server.webserver.responseHeaderSize = 131072 # THE CONFIGURATION FOR Gravitino ENTITY STORE -# The entity store to use +# The entity store to use, kv or relational gravitino.entity.store = kv + # The RocksDB entity store gravitino.entity.store.kv = RocksDBKvBackend # The storage path for RocksDB storage implementation, it supports both absolute and relative path, diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 0d576f71bf9..25628e6206b 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -17,7 +17,6 @@ dependencies { implementation(libs.bundles.prometheus) implementation(libs.caffeine) implementation(libs.commons.io) - implementation(libs.commons.dbcp2) implementation(libs.commons.lang3) implementation(libs.guava) implementation(libs.protobuf.java.util) { @@ -25,6 +24,7 @@ dependencies { .because("Brings in Guava for Android, which we don't want (and breaks multimaps).") } implementation(libs.rocksdbjni) + implementation(libs.commons.dbcp2) implementation(libs.mybatis) annotationProcessor(libs.lombok) diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 1243a219b11..ee39bc04c42 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -22,10 +22,12 @@ public interface Configs { String DEFAULT_ENTITY_RELATIONAL_STORE = "MySQLBackend"; String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; - String MYSQL_ENTITY_STORE_URL_KEY = "gravitino.entity.store.mysql.url"; - String MYSQL_ENTITY_STORE_DRIVER_NAME_KEY = "gravitino.entity.store.mysql.driverName"; - String MYSQL_ENTITY_STORE_USERNAME_KEY = "gravitino.entity.store.mysql.username"; - String MYSQL_ENTITY_STORE_PASSWORD_KEY = "gravitino.entity.store.mysql.password"; + String ENTITY_RELATIONAL_MYSQL_BACKEND_URL_KEY = "gravitino.entity.store.relational.mysqlUrl"; + String ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER_KEY = + "gravitino.entity.store.relational.mysqlDriver"; + String ENTITY_RELATIONAL_MYSQL_BACKEND_USER_KEY = "gravitino.entity.store.relational.mysqlUser"; + String ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD_KEY = + "gravitino.entity.store.relational.mysqlPassword"; String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; @@ -64,35 +66,37 @@ public interface Configs { .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .createWithDefault(DEFAULT_ENTITY_RELATIONAL_STORE); - ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_URL = - new ConfigBuilder(MYSQL_ENTITY_STORE_URL_KEY) + ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_URL = + new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_URL_KEY) .doc("Connection URL of `MySQLBackend`") .version("0.5.0") .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); - ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME = - new ConfigBuilder(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY) + ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER = + new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER_KEY) .doc("Driver Name of `MySQLBackend`") .version("0.5.0") .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); - ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME = - new ConfigBuilder(MYSQL_ENTITY_STORE_USERNAME_KEY) + ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_USER = + new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_USER_KEY) .doc("Username of `MySQLBackend`") .version("0.5.0") .stringConf() - .createWithDefault(""); + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); - ConfigEntry ENTRY_RELATIONAL_MYSQL_BACKEND_PASSWORD = - new ConfigBuilder(MYSQL_ENTITY_STORE_PASSWORD_KEY) + ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD = + new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD_KEY) .doc("Password of `MySQLBackend`") .version("0.5.0") .stringConf() - .createWithDefault(""); + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .create(); ConfigEntry ENTRY_KV_ROCKSDB_BACKEND_PATH = new ConfigBuilder(ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java index 8969db9f912..ea9506ed147 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java @@ -65,10 +65,11 @@ void insert(E e, boolean overwritten) * @param entityType The type of the entity. * @return The entity after updating. * @throws NoSuchEntityException If the entity is not exist. + * @throws IOException If the entity is failed to update. */ E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws NoSuchEntityException; + throws IOException, NoSuchEntityException; /** * Retrieves the entity associated with the identifier and the entity type. diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java index f97cf4c52f8..1944115b364 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java @@ -94,11 +94,7 @@ public E update( public E get( NameIdentifier ident, Entity.EntityType entityType, Class e) throws NoSuchEntityException, IOException { - E entity = backend.get(ident, entityType); - if (entity == null) { - throw new NoSuchEntityException("No such entity:%s", ident.toString()); - } - return entity; + return backend.get(ident, entityType); } @Override diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java index 5816dc0d288..271b6d67d1b 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java @@ -17,17 +17,13 @@ import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessionFactoryHelper; -import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessions; import com.datastrato.gravitino.storage.relational.mysql.utils.POConverters; -import com.fasterxml.jackson.core.JsonProcessingException; +import com.datastrato.gravitino.storage.relational.mysql.utils.SessionUtils; import com.google.common.base.Preconditions; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.function.Function; -import java.util.stream.Collectors; -import org.apache.ibatis.session.SqlSession; /** * {@link MySQLBackend} is a MySQL implementation of RelationalBackend interface. If we want to use @@ -45,134 +41,88 @@ public void initialize(Config config) { @Override public List list( Namespace namespace, Entity.EntityType entityType) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - switch (entityType) { - case METALAKE: - List metalakePOS = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .listMetalakePOs(); - return metalakePOS != null - ? metalakePOS.stream() - .map( - metalakePO -> { - try { - return (E) POConverters.fromMetalakePO(metalakePO); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()) - : new ArrayList<>(); - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for list operation", entityType)); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } finally { - SqlSessions.closeSqlSession(); - } + if (entityType == Entity.EntityType.METALAKE) { + List metalakePOS = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, MetalakeMetaMapper::listMetalakePOs); + return (List) POConverters.fromMetalakePOs(metalakePOS); + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for list operation", entityType)); } } @Override public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - switch (entityType) { - case METALAKE: - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - return metalakePO != null; - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for exists operation", entityType)); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } finally { - SqlSessions.closeSqlSession(); - } + if (entityType == Entity.EntityType.METALAKE) { + MetalakePO metalakePO = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); + return metalakePO != null; + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for exists operation", entityType)); } } @Override public void insert(E e, boolean overwritten) throws EntityAlreadyExistsException { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - if (e instanceof BaseMetalake) { - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(e.nameIdentifier().name()); - if (!overwritten && metalakePO != null) { - throw new EntityAlreadyExistsException( - String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); - } - - if (overwritten) { - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMetaWithUpdate(POConverters.toMetalakePO((BaseMetalake) e)); - } else { - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); - } - SqlSessions.commitAndCloseSqlSession(); - } else { - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for insert operation", e.getClass())); - } - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); + if (e instanceof BaseMetalake) { + MetalakePO metalakePO = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, + mapper -> mapper.selectMetalakeMetaByName(e.nameIdentifier().name())); + if (!overwritten && metalakePO != null) { + throw new EntityAlreadyExistsException( + String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); } + SessionUtils.doWithCommit( + MetalakeMetaMapper.class, + mapper -> { + if (overwritten) { + mapper.insertMetalakeMetaWithUpdate(POConverters.toMetalakePO((BaseMetalake) e)); + } else { + mapper.insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); + } + }); + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for insert operation", e.getClass())); } } @Override public E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws NoSuchEntityException, AlreadyExistsException { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - switch (entityType) { - case METALAKE: - MetalakePO oldMetalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); - BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); - Preconditions.checkArgument( - Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), - String.format( - "The updated metalake entity id: %s should same with the metalake entity id before: %s", - newMetalakeEntity.id(), oldMetalakeEntity.id())); - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO); - SqlSessions.commitAndCloseSqlSession(); - return (E) newMetalakeEntity; - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for update operation", entityType)); - } - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); + throws IOException, NoSuchEntityException, AlreadyExistsException { + if (entityType == Entity.EntityType.METALAKE) { + MetalakePO oldMetalakePO = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); + + BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); + BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); + Preconditions.checkArgument( + Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), + String.format( + "The updated metalake entity id: %s should same with the metalake entity id before: %s", + newMetalakeEntity.id(), oldMetalakeEntity.id())); + + Integer updateResult = + SessionUtils.doWithCommitAndFetchResult( + MetalakeMetaMapper.class, + mapper -> + mapper.updateMetalakeMeta( + POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO)); + if (updateResult > 0) { + return (E) newMetalakeEntity; + } else { + throw new IOException("Failed to update the entity:" + ident); } + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for update operation", entityType)); } } @@ -180,67 +130,50 @@ public E update( public E get( NameIdentifier ident, Entity.EntityType entityType) throws NoSuchEntityException, IOException { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - switch (entityType) { - case METALAKE: - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - if (metalakePO == null) { - return null; - } - return (E) POConverters.fromMetalakePO(metalakePO); - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for get operation", entityType)); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } finally { - SqlSessions.closeSqlSession(); + if (entityType == Entity.EntityType.METALAKE) { + MetalakePO metalakePO = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); + if (metalakePO == null) { + throw new NoSuchEntityException("No such entity:%s", ident.toString()); } + return (E) POConverters.fromMetalakePO(metalakePO); + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for get operation", entityType)); } } @Override public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - switch (entityType) { - case METALAKE: - Long metalakeId = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeIdMetaByName(ident.name()); - if (metalakeId != null) { - // delete metalake - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .deleteMetalakeMetaById(metalakeId); - if (cascade) { + if (entityType == Entity.EntityType.METALAKE) { + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(ident.name())); + if (metalakeId != null) { + if (cascade) { + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + MetalakeMetaMapper.class, + mapper -> mapper.deleteMetalakeMetaById(metalakeId)), + () -> { // TODO We will cascade delete the metadata of sub-resources under the metalake - } - SqlSessions.commitAndCloseSqlSession(); - } - return true; - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for delete operation", entityType)); + }); + } else { + SessionUtils.doWithCommit( + MetalakeMetaMapper.class, mapper -> mapper.deleteMetalakeMetaById(metalakeId)); } - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); } + return true; + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for delete operation", entityType)); } } @Override - public void close() throws IOException {} + public void close() throws IOException { + SqlSessionFactoryHelper.getInstance().close(); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java index f2791a22558..7744c6633c8 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java @@ -13,6 +13,14 @@ import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.Update; +/** + * A MyBatis Mapper for metalake meta operation SQLs. + * + *

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQL in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ public interface MetalakeMetaMapper { String TABLE_NAME = "metalake_meta"; @@ -83,7 +91,7 @@ public interface MetalakeMetaMapper { + " and properties = #{oldMetalakeMeta.properties}" + " and audit_info = #{oldMetalakeMeta.auditInfo}" + " and schema_version = #{oldMetalakeMeta.schemaVersion}") - void updateMetalakeMeta( + Integer updateMetalakeMeta( @Param("newMetalakeMeta") MetalakePO newMetalakePO, @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java index d372cd045b7..7569d32eb13 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java @@ -10,6 +10,7 @@ import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import java.sql.SQLException; import java.time.Duration; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.pool2.impl.BaseObjectPoolConfig; @@ -47,12 +48,13 @@ static void setSqlSessionFactory(SqlSessionFactory sessionFactory) { */ @SuppressWarnings("deprecation") public void init(Config config) { + // Initialize the data source BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_URL)); - dataSource.setDriverClassName(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME)); - dataSource.setUsername(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME)); - dataSource.setPassword(config.get(Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_PASSWORD)); - // close the auto commit, so that need manual commit + dataSource.setUrl(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_URL)); + dataSource.setDriverClassName(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER)); + dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_USER)); + dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD)); + // Close the auto commit, so that we can control the transaction manual commit dataSource.setDefaultAutoCommit(false); dataSource.setMaxWaitMillis(1000L); dataSource.setMaxTotal(20); @@ -70,10 +72,16 @@ public void init(Config config) { dataSource.setSoftMinEvictableIdleTimeMillis( BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME.toMillis()); dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO); + + // Create the transaction factory and env TransactionFactory transactionFactory = new JdbcTransactionFactory(); Environment environment = new Environment("development", transactionFactory, dataSource); + + // Initialize the configuration Configuration configuration = new Configuration(environment); configuration.addMapper(MetalakeMetaMapper.class); + + // Create the SqlSessionFactory object, it is a singleton object if (sqlSessionFactory == null) { synchronized (SqlSessionFactoryHelper.class) { if (sqlSessionFactory == null) { @@ -87,4 +95,16 @@ public SqlSessionFactory getSqlSessionFactory() { Preconditions.checkState(sqlSessionFactory != null, "SqlSessionFactory is not initialized."); return sqlSessionFactory; } + + public void close() { + if (sqlSessionFactory != null) { + try { + BasicDataSource dataSource = + (BasicDataSource) sqlSessionFactory.getConfiguration().getEnvironment().getDataSource(); + dataSource.close(); + } catch (SQLException e) { + // silently ignore the error report + } + } + } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java index 405651aecf2..e018ba929ef 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java @@ -11,10 +11,13 @@ import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** POConverters is a utility class to convert PO to Base and vice versa. */ public class POConverters { + private POConverters() {} /** @@ -22,17 +25,20 @@ private POConverters() {} * * @param baseMetalake BaseMetalake object * @return MetalakePO object from BaseMetalake object - * @throws JsonProcessingException */ - public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) throws JsonProcessingException { - return new MetalakePO.Builder() - .withId(baseMetalake.id()) - .withMetalakeName(baseMetalake.name()) - .withMetalakeComment(baseMetalake.comment()) - .withProperties(JsonUtils.objectMapper().writeValueAsString(baseMetalake.properties())) - .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(baseMetalake.auditInfo())) - .withVersion(JsonUtils.objectMapper().writeValueAsString(baseMetalake.getVersion())) - .build(); + public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { + try { + return new MetalakePO.Builder() + .withId(baseMetalake.id()) + .withMetalakeName(baseMetalake.name()) + .withMetalakeComment(baseMetalake.comment()) + .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.properties())) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.auditInfo())) + .withVersion(JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.getVersion())) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } } /** @@ -40,18 +46,33 @@ public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) throws JsonProc * * @param metalakePO MetalakePO object * @return BaseMetalake object from MetalakePO object - * @throws JsonProcessingException */ - public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) throws JsonProcessingException { - return new BaseMetalake.Builder() - .withId(metalakePO.getId()) - .withName(metalakePO.getMetalakeName()) - .withComment(metalakePO.getMetalakeComment()) - .withProperties(JsonUtils.objectMapper().readValue(metalakePO.getProperties(), Map.class)) - .withAuditInfo( - JsonUtils.objectMapper().readValue(metalakePO.getAuditInfo(), AuditInfo.class)) - .withVersion( - JsonUtils.objectMapper().readValue(metalakePO.getSchemaVersion(), SchemaVersion.class)) - .build(); + public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) { + try { + return new BaseMetalake.Builder() + .withId(metalakePO.getId()) + .withName(metalakePO.getMetalakeName()) + .withComment(metalakePO.getMetalakeComment()) + .withProperties( + JsonUtils.anyFieldMapper().readValue(metalakePO.getProperties(), Map.class)) + .withAuditInfo( + JsonUtils.anyFieldMapper().readValue(metalakePO.getAuditInfo(), AuditInfo.class)) + .withVersion( + JsonUtils.anyFieldMapper() + .readValue(metalakePO.getSchemaVersion(), SchemaVersion.class)) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to deserialize json object:", e); + } + } + + /** + * Convert list of {@link MetalakePO} to list of {@link BaseMetalake} + * + * @param metalakePOS list of MetalakePO objects + * @return list of BaseMetalake objects from list of MetalakePO objects + */ + public static List fromMetalakePOs(List metalakePOS) { + return metalakePOS.stream().map(POConverters::fromMetalakePO).collect(Collectors.toList()); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java new file mode 100644 index 00000000000..2b6869f9440 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java @@ -0,0 +1,78 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.utils; + +import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessions; +import java.util.Arrays; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.ibatis.session.SqlSession; + +public class SessionUtils { + private SessionUtils() {} + + public static void doWithCommit(Class mapperClazz, Consumer consumer) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + consumer.accept(mapper); + SqlSessions.commitAndCloseSqlSession(); + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } + } + + public static R doWithCommitAndFetchResult(Class mapperClazz, Function func) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + R result = func.apply(mapper); + SqlSessions.commitAndCloseSqlSession(); + return result; + } catch (Throwable t) { + throw new RuntimeException(t); + } finally { + SqlSessions.closeSqlSession(); + } + } + } + + public static void doWithoutCommit(Class mapperClazz, Consumer consumer) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + consumer.accept(mapper); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + + public static R getWithoutCommit(Class mapperClazz, Function func) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + return func.apply(mapper); + } catch (Throwable t) { + throw new RuntimeException(t); + } finally { + SqlSessions.closeSqlSession(); + } + } + } + + public static void doMultipleWithCommit(Runnable... operations) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + Arrays.stream(operations).forEach(Runnable::run); + SqlSessions.commitAndCloseSqlSession(); + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } + } +} diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql index 610b812a2b9..bb068a416bc 100644 --- a/core/src/main/resources/mysql/mysql_init.sql +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -3,8 +3,7 @@ -- This software is licensed under the Apache License version 2. -- -CREATE TABLE IF NOT EXISTS `metalake_meta` -( +CREATE TABLE IF NOT EXISTS `metalake_meta` ( `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', @@ -13,4 +12,4 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', PRIMARY KEY (`id`), UNIQUE KEY `uk_mn` (`metalake_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'metalake metadata'; \ No newline at end of file +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index ecbe0fb46ad..6b67877872a 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -6,11 +6,11 @@ package com.datastrato.gravitino.storage.relational; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_USER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME; -import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_URL; -import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -69,11 +69,10 @@ public static void setUp() { Config config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_URL)) + Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME)).thenReturn("sa"); - Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME)) - .thenReturn("org.h2.Driver"); + Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_USER)).thenReturn("sa"); + Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); entityStore = EntityStoreFactory.createEntityStore(config); entityStore.initialize(config); @@ -148,6 +147,11 @@ public void testPutAndGet() throws IOException { public void testPutAndList() throws IOException { BaseMetalake metalake1 = createMetalake(1L, "test_metalake1", "this is test 1"); BaseMetalake metalake2 = createMetalake(2L, "test_metalake2", "this is test 2"); + List beforePutList = + entityStore.list(metalake1.namespace(), BaseMetalake.class, Entity.EntityType.METALAKE); + assertNotNull(beforePutList); + assertEquals(0, beforePutList.size()); + entityStore.put(metalake1, false); entityStore.put(metalake2, false); List metalakes = diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java index 299dd65a6f7..01ab871e804 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java @@ -6,11 +6,11 @@ package com.datastrato.gravitino.storage.relational.mysql.session; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_USER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME; -import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_URL; -import static com.datastrato.gravitino.Configs.ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -49,11 +49,10 @@ public static void setUp() { config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_URL)) + Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_USERNAME)).thenReturn("sa"); - Mockito.when(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_DRIVER_NAME)) - .thenReturn("org.h2.Driver"); + Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_USER)).thenReturn("sa"); + Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); } @BeforeEach @@ -94,7 +93,7 @@ public void testInit() throws SQLException { .getEnvironment() .getDataSource(); assertEquals("org.h2.Driver", dataSource.getDriverClassName()); - assertEquals(config.get(ENTRY_RELATIONAL_MYSQL_BACKEND_URL), dataSource.getUrl()); + assertEquals(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_URL), dataSource.getUrl()); } @Test diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java index 3612f099466..27f49756626 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java @@ -85,9 +85,9 @@ private static MetalakePO createMetalakePO(Long id, String name, String comment) .withId(id) .withMetalakeName(name) .withMetalakeComment(comment) - .withProperties(JsonUtils.objectMapper().writeValueAsString(properties)) - .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(auditInfo)) - .withVersion(JsonUtils.objectMapper().writeValueAsString(SchemaVersion.V_0_1)) + .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(properties)) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(auditInfo)) + .withVersion(JsonUtils.anyFieldMapper().writeValueAsString(SchemaVersion.V_0_1)) .build(); } } diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index a481db94ba0..c8e5422a767 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -4,12 +4,12 @@ -- CREATE TABLE IF NOT EXISTS `metalake_meta` ( - `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', - `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', - `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', - PRIMARY KEY (id), + `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', + `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', + PRIMARY KEY (id), CONSTRAINT uk_mn UNIQUE (metalake_name) ) ENGINE = InnoDB; \ No newline at end of file diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 7a4072b8db3..1a1c6cfa823 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -51,6 +51,11 @@ You can also specify filter parameters by setting configuration entries of the f | `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | | `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | | `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | +| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` storage is currently supported, and the implementation is `MySQLBackend`. | `MySQLBackend` | No | 0.5.0 | +| `gravitino.entity.store.relational.mysqlUrl` | The database URL that the `MySQLBackend` needs to connect to. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.mysqlDriver` | The jdbc driver name that the `MySQLBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.mysqlUser` | The username that the `MySQLBackend` needs to use when connecting the database. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.mysqlPassword` | The password that the `MySQLBackend` needs to use when connecting the database. | (none) | Yes | 0.5.0 | :::caution We strongly recommend that you change the default value of `gravitino.entity.store.kv.rocksdbPath`, as it's under the deployment directory and future version upgrades may remove it. From ff736d8576db6dfeea7f0e36503b1a8fbf665961 Mon Sep 17 00:00:00 2001 From: Jiebao Xiao Date: Sat, 17 Feb 2024 14:43:52 +0800 Subject: [PATCH 11/34] rename to jdbc backend --- .../com/datastrato/gravitino/Configs.java | 47 +++++++++++-------- .../MySQLBackend.java => JDBCBackend.java} | 20 ++++---- .../relational/RelationalEntityStore.java | 3 +- .../mapper/MetalakeMetaMapper.java | 4 +- .../relational/{mysql => }/po/MetalakePO.java | 2 +- .../session/SqlSessionFactoryHelper.java | 35 +++++++++----- .../{mysql => }/session/SqlSessions.java | 2 +- .../{mysql => }/utils/POConverters.java | 4 +- .../{mysql => }/utils/SessionUtils.java | 4 +- .../relational/TestRelationalEntityStore.java | 20 +++++--- .../{mysql => }/session/TestSqlSession.java | 24 ++++++---- .../{mysql => }/utils/TestPOConverters.java | 4 +- docs/gravitino-server-config.md | 10 ++-- 13 files changed, 104 insertions(+), 75 deletions(-) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{mysql/MySQLBackend.java => JDBCBackend.java} (89%) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{mysql => }/mapper/MetalakeMetaMapper.java (96%) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{mysql => }/po/MetalakePO.java (97%) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{mysql => }/session/SqlSessionFactoryHelper.java (77%) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{mysql => }/session/SqlSessions.java (97%) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{mysql => }/utils/POConverters.java (95%) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{mysql => }/utils/SessionUtils.java (94%) rename core/src/test/java/com/datastrato/gravitino/storage/relational/{mysql => }/session/TestSqlSession.java (80%) rename core/src/test/java/com/datastrato/gravitino/storage/relational/{mysql => }/utils/TestPOConverters.java (96%) diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index ee39bc04c42..06faf385352 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -19,15 +19,16 @@ public interface Configs { String DEFAULT_ENTITY_KV_STORE = "RocksDBKvBackend"; String ENTITY_KV_STORE_KEY = "gravitino.entity.store.kv"; - String DEFAULT_ENTITY_RELATIONAL_STORE = "MySQLBackend"; + String DEFAULT_ENTITY_RELATIONAL_STORE = "JDBCBackend"; String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; - String ENTITY_RELATIONAL_MYSQL_BACKEND_URL_KEY = "gravitino.entity.store.relational.mysqlUrl"; - String ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER_KEY = - "gravitino.entity.store.relational.mysqlDriver"; - String ENTITY_RELATIONAL_MYSQL_BACKEND_USER_KEY = "gravitino.entity.store.relational.mysqlUser"; - String ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD_KEY = - "gravitino.entity.store.relational.mysqlPassword"; + String ENTITY_RELATIONAL_JDBC_BACKEND_TYPE_KEY = "gravitino.entity.store.relational.jdbcType"; + String DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE = "mysql"; + String ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY = "gravitino.entity.store.relational.jdbcUrl"; + String ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY = "gravitino.entity.store.relational.jdbcDriver"; + String ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY = "gravitino.entity.store.relational.jdbcUser"; + String ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY = + "gravitino.entity.store.relational.jdbcPassword"; String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; @@ -66,33 +67,41 @@ public interface Configs { .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .createWithDefault(DEFAULT_ENTITY_RELATIONAL_STORE); - ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_URL = - new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_URL_KEY) - .doc("Connection URL of `MySQLBackend`") + ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_TYPE = + new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_TYPE_KEY) + .doc("Database type of `JDBCBackend`") + .version("0.5.0") + .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) + .createWithDefault(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); + + ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_URL = + new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY) + .doc("Connection URL of `JDBCBackend`") .version("0.5.0") .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); - ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER = - new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER_KEY) - .doc("Driver Name of `MySQLBackend`") + ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER = + new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY) + .doc("Driver Name of `JDBCBackend`") .version("0.5.0") .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); - ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_USER = - new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_USER_KEY) - .doc("Username of `MySQLBackend`") + ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_USER = + new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY) + .doc("Username of `JDBCBackend`") .version("0.5.0") .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); - ConfigEntry ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD = - new ConfigBuilder(ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD_KEY) - .doc("Password of `MySQLBackend`") + ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD = + new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY) + .doc("Password of `JDBCBackend`") .version("0.5.0") .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java similarity index 89% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 271b6d67d1b..24987290b93 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -2,9 +2,10 @@ * Copyright 2024 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql; +package com.datastrato.gravitino.storage.relational; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; @@ -13,12 +14,11 @@ import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.BaseMetalake; -import com.datastrato.gravitino.storage.relational.RelationalBackend; -import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; -import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; -import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessionFactoryHelper; -import com.datastrato.gravitino.storage.relational.mysql.utils.POConverters; -import com.datastrato.gravitino.storage.relational.mysql.utils.SessionUtils; +import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relational.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import com.datastrato.gravitino.storage.relational.utils.POConverters; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; import com.google.common.base.Preconditions; import java.io.IOException; import java.util.List; @@ -26,13 +26,13 @@ import java.util.function.Function; /** - * {@link MySQLBackend} is a MySQL implementation of RelationalBackend interface. If we want to use + * {@link JDBCBackend} is a jdbc implementation of RelationalBackend interface. If we want to use * another relational implementation, We can just implement {@link RelationalBackend} interface and * use it in the Gravitino. */ -public class MySQLBackend implements RelationalBackend { +public class JDBCBackend implements RelationalBackend { - /** Initialize the MySQL backend instance. */ + /** Initialize the jdbc backend instance. */ @Override public void initialize(Config config) { SqlSessionFactoryHelper.getInstance().init(config); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java index 1944115b364..8427e7484af 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java @@ -17,7 +17,6 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.storage.relational.mysql.MySQLBackend; import com.datastrato.gravitino.utils.Executable; import com.google.common.collect.ImmutableMap; import java.io.IOException; @@ -35,7 +34,7 @@ public class RelationalEntityStore implements EntityStore { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); public static final ImmutableMap RELATIONAL_BACKENDS = ImmutableMap.of( - Configs.DEFAULT_ENTITY_RELATIONAL_STORE, MySQLBackend.class.getCanonicalName()); + Configs.DEFAULT_ENTITY_RELATIONAL_STORE, JDBCBackend.class.getCanonicalName()); private RelationalBackend backend; @Override diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java similarity index 96% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index 7744c6633c8..43d7d88f317 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -3,9 +3,9 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.mapper; +package com.datastrato.gravitino.storage.relational.mapper; -import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.po.MetalakePO; import java.util.List; import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/MetalakePO.java similarity index 97% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/po/MetalakePO.java index 8b36eeace56..9d833bb4861 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/po/MetalakePO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/MetalakePO.java @@ -3,7 +3,7 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.po; +package com.datastrato.gravitino.storage.relational.po; import com.google.common.base.Objects; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java similarity index 77% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index 7569d32eb13..b833a4e5110 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -3,11 +3,11 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.session; +package com.datastrato.gravitino.storage.relational.session; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; -import com.datastrato.gravitino.storage.relational.mysql.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.sql.SQLException; @@ -44,16 +44,19 @@ static void setSqlSessionFactory(SqlSessionFactory sessionFactory) { /** * Initialize the SqlSessionFactory object. * - * @param config Config object to get the MySQL connection details from the config. + * @param config Config object to get the jdbc connection details from the config. */ @SuppressWarnings("deprecation") public void init(Config config) { // Initialize the data source BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_URL)); - dataSource.setDriverClassName(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER)); - dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_USER)); - dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_PASSWORD)); + dataSource.setUrl(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL)); + dataSource.setDriverClassName(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)); + String dbType = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); + if (dbType.equals(Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE)) { + dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER)); + dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)); + } // Close the auto commit, so that we can control the transaction manual commit dataSource.setDefaultAutoCommit(false); dataSource.setMaxWaitMillis(1000L); @@ -98,12 +101,18 @@ public SqlSessionFactory getSqlSessionFactory() { public void close() { if (sqlSessionFactory != null) { - try { - BasicDataSource dataSource = - (BasicDataSource) sqlSessionFactory.getConfiguration().getEnvironment().getDataSource(); - dataSource.close(); - } catch (SQLException e) { - // silently ignore the error report + synchronized (SqlSessionFactoryHelper.class) { + if (sqlSessionFactory != null) { + try { + BasicDataSource dataSource = + (BasicDataSource) + sqlSessionFactory.getConfiguration().getEnvironment().getDataSource(); + dataSource.close(); + } catch (SQLException e) { + // silently ignore the error report + } + sqlSessionFactory = null; + } } } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java similarity index 97% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java index 044ed72c093..bd10c21fb80 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/session/SqlSessions.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java @@ -3,7 +3,7 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.session; +package com.datastrato.gravitino.storage.relational.session; import com.google.common.annotations.VisibleForTesting; import org.apache.ibatis.session.SqlSession; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java similarity index 95% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index e018ba929ef..1dbf3cbf7cc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -3,13 +3,13 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.utils; +package com.datastrato.gravitino.storage.relational.utils; import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java similarity index 94% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java index 2b6869f9440..530e5cc2e10 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java @@ -3,9 +3,9 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.utils; +package com.datastrato.gravitino.storage.relational.utils; -import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessions; +import com.datastrato.gravitino.storage.relational.session.SqlSessions; import java.util.Arrays; import java.util.function.Consumer; import java.util.function.Function; diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index 6b67877872a..826a785247f 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -5,10 +5,13 @@ package com.datastrato.gravitino.storage.relational; +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_URL; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_USER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; @@ -26,7 +29,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessionFactoryHelper; +import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -69,10 +72,13 @@ public static void setUp() { Config config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_URL)) + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_TYPE)) + .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_USER)).thenReturn("sa"); - Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("123"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); entityStore = EntityStoreFactory.createEntityStore(config); entityStore.initialize(config); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java similarity index 80% rename from core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java rename to core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java index 01ab871e804..e4bf8cffd24 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSession.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java @@ -3,12 +3,15 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.session; +package com.datastrato.gravitino.storage.relational.session; +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_URL; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_MYSQL_BACKEND_USER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; @@ -49,10 +52,13 @@ public static void setUp() { config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_URL)) + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_TYPE)) + .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_USER)).thenReturn("sa"); - Mockito.when(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("123"); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); } @BeforeEach @@ -62,7 +68,7 @@ public void init() { @AfterEach public void cleanUp() { - SqlSessionFactoryHelper.setSqlSessionFactory(null); + SqlSessionFactoryHelper.getInstance().close(); } @AfterAll @@ -93,7 +99,7 @@ public void testInit() throws SQLException { .getEnvironment() .getDataSource(); assertEquals("org.h2.Driver", dataSource.getDriverClassName()); - assertEquals(config.get(ENTITY_RELATIONAL_MYSQL_BACKEND_URL), dataSource.getUrl()); + assertEquals(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL), dataSource.getUrl()); } @Test diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java similarity index 96% rename from core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java rename to core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java index 27f49756626..fc10c48dbcb 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java @@ -3,7 +3,7 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.storage.relational.mysql.utils; +package com.datastrato.gravitino.storage.relational.utils; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -11,7 +11,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; import java.time.Instant; import java.time.LocalDateTime; diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 1a1c6cfa823..007959f7515 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -51,11 +51,11 @@ You can also specify filter parameters by setting configuration entries of the f | `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | | `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | | `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | -| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` storage is currently supported, and the implementation is `MySQLBackend`. | `MySQLBackend` | No | 0.5.0 | -| `gravitino.entity.store.relational.mysqlUrl` | The database URL that the `MySQLBackend` needs to connect to. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.mysqlDriver` | The jdbc driver name that the `MySQLBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.mysqlUser` | The username that the `MySQLBackend` needs to use when connecting the database. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.mysqlPassword` | The password that the `MySQLBackend` needs to use when connecting the database. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.jdbcType` | The database type that the `JDBCBackend` needs to connect to. | `mysql` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | :::caution We strongly recommend that you change the default value of `gravitino.entity.store.kv.rocksdbPath`, as it's under the deployment directory and future version upgrades may remove it. From 146256d12eacdf38724a8ed167c35edc92244ede Mon Sep 17 00:00:00 2001 From: Jiebao Xiao Date: Sat, 17 Feb 2024 17:13:36 +0800 Subject: [PATCH 12/34] fix comments --- .../com/datastrato/gravitino/Configs.java | 10 +++---- .../storage/relational/JDBCBackend.java | 1 - .../relational/mapper/MetalakeMetaMapper.java | 2 +- .../relational/session/SqlSessions.java | 5 ++-- .../relational/utils/SessionUtils.java | 3 +-- docs/gravitino-server-config.md | 27 ++++++++++--------- 6 files changed, 23 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 06faf385352..f7d2d7af717 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -70,7 +70,7 @@ public interface Configs { ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_TYPE = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_TYPE_KEY) .doc("Database type of `JDBCBackend`") - .version("0.5.0") + .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .createWithDefault(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); @@ -78,7 +78,7 @@ public interface Configs { ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_URL = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY) .doc("Connection URL of `JDBCBackend`") - .version("0.5.0") + .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); @@ -86,7 +86,7 @@ public interface Configs { ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY) .doc("Driver Name of `JDBCBackend`") - .version("0.5.0") + .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); @@ -94,7 +94,7 @@ public interface Configs { ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_USER = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY) .doc("Username of `JDBCBackend`") - .version("0.5.0") + .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); @@ -102,7 +102,7 @@ public interface Configs { ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY) .doc("Password of `JDBCBackend`") - .version("0.5.0") + .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 24987290b93..863fc272083 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -5,7 +5,6 @@ package com.datastrato.gravitino.storage.relational; import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index 43d7d88f317..104c83469fc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -17,7 +17,7 @@ * A MyBatis Mapper for metalake meta operation SQLs. * *

This interface class is a specification defined by MyBatis. It requires this interface class - * to identify the corresponding SQLs for execution. We can write SQL in an additional XML file, or + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or * write SQLs with annotations in this interface Mapper. See: */ diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java index bd10c21fb80..ca3684981a0 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java @@ -83,9 +83,8 @@ public static void closeSqlSession() { * * @param className the class name of the Mapper object. * @return the Mapper object. - * @param the type of the Mapper object. */ - public static T getMapper(Class className) { - return (T) getSqlSession().getMapper(className); + public static T getMapper(Class className) { + return getSqlSession().getMapper(className); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java index 530e5cc2e10..62ef8243afd 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java @@ -35,9 +35,8 @@ public static R doWithCommitAndFetchResult(Class mapperClazz, Function SqlSessions.commitAndCloseSqlSession(); return result; } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); throw new RuntimeException(t); - } finally { - SqlSessions.closeSqlSession(); } } } diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 007959f7515..f579ba63be6 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -43,19 +43,20 @@ You can also specify filter parameters by setting configuration entries of the f ### Storage configuration -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|----------|---------------| -| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | -| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | -| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | -| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | -| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | -| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | -| `gravitino.entity.store.relational.jdbcType` | The database type that the `JDBCBackend` needs to connect to. | `mysql` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|----------|---------------| +| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | +| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | +| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | +| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | +| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | +| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | +| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` storage is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbcType` | The database type that the `JDBCBackend` needs to connect to. If you use `MySQL` storage, you should first initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | `mysql` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | +| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | :::caution We strongly recommend that you change the default value of `gravitino.entity.store.kv.rocksdbPath`, as it's under the deployment directory and future version upgrades may remove it. From 45c88b27c5d38863df709307baff03797856a4b2 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Sun, 18 Feb 2024 11:38:51 +0800 Subject: [PATCH 13/34] fix comments --- .../com/datastrato/gravitino/Configs.java | 20 +++++++------ .../storage/relational/JDBCBackend.java | 8 ++++-- .../session/SqlSessionFactoryHelper.java | 4 +-- .../relational/TestRelationalEntityStore.java | 8 +++--- .../relational/session/TestSqlSession.java | 8 +++--- docs/gravitino-server-config.md | 28 +++++++++---------- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index f7d2d7af717..0135e30206d 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -22,13 +22,15 @@ public interface Configs { String DEFAULT_ENTITY_RELATIONAL_STORE = "JDBCBackend"; String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; - String ENTITY_RELATIONAL_JDBC_BACKEND_TYPE_KEY = "gravitino.entity.store.relational.jdbcType"; - String DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE = "mysql"; - String ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY = "gravitino.entity.store.relational.jdbcUrl"; - String ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY = "gravitino.entity.store.relational.jdbcDriver"; - String ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY = "gravitino.entity.store.relational.jdbcUser"; + String ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE_KEY = + "gravitino.entity.store.relational.jdbc.dbType"; + String DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE = "mysql"; + String ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY = "gravitino.entity.store.relational.jdbc.url"; + String ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY = + "gravitino.entity.store.relational.jdbc.driver"; + String ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY = "gravitino.entity.store.relational.jdbc.user"; String ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY = - "gravitino.entity.store.relational.jdbcPassword"; + "gravitino.entity.store.relational.jdbc.password"; String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; @@ -67,13 +69,13 @@ public interface Configs { .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .createWithDefault(DEFAULT_ENTITY_RELATIONAL_STORE); - ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_TYPE = - new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_TYPE_KEY) + ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE = + new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE_KEY) .doc("Database type of `JDBCBackend`") .version(ConfigConstants.VERSION_0_5_0) .stringConf() .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .createWithDefault(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); + .createWithDefault(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_URL = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 863fc272083..50b79db8498 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.storage.relational; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; @@ -25,9 +26,10 @@ import java.util.function.Function; /** - * {@link JDBCBackend} is a jdbc implementation of RelationalBackend interface. If we want to use - * another relational implementation, We can just implement {@link RelationalBackend} interface and - * use it in the Gravitino. + * {@link JDBCBackend} is a jdbc implementation of {@link RelationalBackend} interface. You can use + * a database that supports the JDBC protocol as storage. If the specified database has special SQL + * syntax, please implement the SQL statements and methods in MyBatis Mapper separately and switch + * according to the {@link Configs#ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE_KEY} parameter. */ public class JDBCBackend implements RelationalBackend { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index b833a4e5110..8a9ad4581bc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -52,8 +52,8 @@ public void init(Config config) { BasicDataSource dataSource = new BasicDataSource(); dataSource.setUrl(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL)); dataSource.setDriverClassName(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)); - String dbType = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); - if (dbType.equals(Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE)) { + String dbType = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); + if (dbType.equals(Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE)) { dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER)); dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)); } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index 826a785247f..808ac5a1c3a 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -5,11 +5,11 @@ package com.datastrato.gravitino.storage.relational; -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; @@ -72,8 +72,8 @@ public static void setUp() { Config config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_TYPE)) - .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE)) + .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java index e4bf8cffd24..b6af9553a23 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java @@ -5,11 +5,11 @@ package com.datastrato.gravitino.storage.relational.session; -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_TYPE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; @@ -52,8 +52,8 @@ public static void setUp() { config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_TYPE)) - .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_TYPE); + Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE)) + .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index f579ba63be6..59a48338e74 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -43,20 +43,20 @@ You can also specify filter parameters by setting configuration entries of the f ### Storage configuration -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|----------|---------------| -| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | -| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | -| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | -| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | -| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | -| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | -| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` storage is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbcType` | The database type that the `JDBCBackend` needs to connect to. If you use `MySQL` storage, you should first initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | `mysql` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | -| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes | 0.5.0 | +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|---------------------------------------------------|---------------| +| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | +| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | +| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | +| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | +| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | +| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | +| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` storage is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbc.dbType` | The database type that the `JDBCBackend` needs to connect to. If you use `MySQL` storage, you should first initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | `mysql` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbc.url` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbc.driver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbc.user` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` storage | 0.5.0 | +| `gravitino.entity.store.relational.jdbc.password` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` storage | 0.5.0 | :::caution We strongly recommend that you change the default value of `gravitino.entity.store.kv.rocksdbPath`, as it's under the deployment directory and future version upgrades may remove it. From 5a45836262dd36f15fb8bd5afd5083e6d2f4febc Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Sun, 18 Feb 2024 13:03:42 +0800 Subject: [PATCH 14/34] fix comments --- .../com/datastrato/gravitino/Configs.java | 11 ++++---- core/src/test/resources/h2/h2-init.sql | 14 +++++----- docs/gravitino-server-config.md | 28 +++++++++---------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 0135e30206d..48bbb3087f7 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -23,14 +23,13 @@ public interface Configs { String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; String ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE_KEY = - "gravitino.entity.store.relational.jdbc.dbType"; + "gravitino.entity.store.relational.jdbcDBType"; String DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE = "mysql"; - String ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY = "gravitino.entity.store.relational.jdbc.url"; - String ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY = - "gravitino.entity.store.relational.jdbc.driver"; - String ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY = "gravitino.entity.store.relational.jdbc.user"; + String ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY = "gravitino.entity.store.relational.jdbcUrl"; + String ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY = "gravitino.entity.store.relational.jdbcDriver"; + String ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY = "gravitino.entity.store.relational.jdbcUser"; String ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY = - "gravitino.entity.store.relational.jdbc.password"; + "gravitino.entity.store.relational.jdbcPassword"; String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index c8e5422a767..af83ecdf197 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -4,12 +4,12 @@ -- CREATE TABLE IF NOT EXISTS `metalake_meta` ( - `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', - `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', - `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', - PRIMARY KEY (id), + `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', + `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', + PRIMARY KEY (id), CONSTRAINT uk_mn UNIQUE (metalake_name) ) ENGINE = InnoDB; \ No newline at end of file diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 59a48338e74..93c2a0caa50 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -43,20 +43,20 @@ You can also specify filter parameters by setting configuration entries of the f ### Storage configuration -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|---------------------------------------------------|---------------| -| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | -| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | -| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | -| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | -| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | -| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | -| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` storage is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbc.dbType` | The database type that the `JDBCBackend` needs to connect to. If you use `MySQL` storage, you should first initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | `mysql` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbc.url` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | -| `gravitino.entity.store.relational.jdbc.driver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | -| `gravitino.entity.store.relational.jdbc.user` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` storage | 0.5.0 | -| `gravitino.entity.store.relational.jdbc.password` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` storage | 0.5.0 | +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|--------------------------------------------|---------------| +| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | +| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | +| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | +| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | +| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | +| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | +| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbcDBType` | The database type that the `JDBCBackend` needs to connect to. If you use `MySQL`, you should firstly initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | `mysql` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` | 0.5.0 | :::caution We strongly recommend that you change the default value of `gravitino.entity.store.kv.rocksdbPath`, as it's under the deployment directory and future version upgrades may remove it. From 956b5a61927d5f1092d398bd7a6e0fa986c4444d Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Sun, 18 Feb 2024 16:19:18 +0800 Subject: [PATCH 15/34] fix comments --- .../UnsupportedEntityTypeException.java | 36 +++++++++ .../storage/relational/JDBCBackend.java | 73 +++++++++++-------- .../relational/mapper/MetalakeMetaMapper.java | 2 +- .../relational/TestRelationalEntityStore.java | 16 +++- 4 files changed, 92 insertions(+), 35 deletions(-) create mode 100644 api/src/main/java/com/datastrato/gravitino/exceptions/UnsupportedEntityTypeException.java diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/UnsupportedEntityTypeException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/UnsupportedEntityTypeException.java new file mode 100644 index 00000000000..9e6aaa55d34 --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/UnsupportedEntityTypeException.java @@ -0,0 +1,36 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when an entity type is not supported for operations. */ +public class UnsupportedEntityTypeException extends GravitinoRuntimeException { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public UnsupportedEntityTypeException(@FormatString String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public UnsupportedEntityTypeException( + Throwable cause, @FormatString String message, Object... args) { + super(cause, message, args); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 50b79db8498..d50787ad273 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -13,6 +13,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.exceptions.UnsupportedEntityTypeException; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.datastrato.gravitino.storage.relational.po.MetalakePO; @@ -21,6 +22,7 @@ import com.datastrato.gravitino.storage.relational.utils.SessionUtils; import com.google.common.base.Preconditions; import java.io.IOException; +import java.sql.SQLIntegrityConstraintViolationException; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -48,8 +50,8 @@ public List list( MetalakeMetaMapper.class, MetalakeMetaMapper::listMetalakePOs); return (List) POConverters.fromMetalakePOs(metalakePOS); } else { - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for list operation", entityType)); + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for list operation", entityType); } } @@ -61,8 +63,8 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); return metalakePO != null; } else { - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for exists operation", entityType)); + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for exists operation", entityType); } } @@ -70,26 +72,33 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { public void insert(E e, boolean overwritten) throws EntityAlreadyExistsException { if (e instanceof BaseMetalake) { - MetalakePO metalakePO = - SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, - mapper -> mapper.selectMetalakeMetaByName(e.nameIdentifier().name())); - if (!overwritten && metalakePO != null) { - throw new EntityAlreadyExistsException( - String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); + try { + SessionUtils.doWithCommit( + MetalakeMetaMapper.class, + mapper -> { + if (overwritten) { + mapper.insertMetalakeMetaOnDuplicateKeyUpdate( + POConverters.toMetalakePO((BaseMetalake) e)); + } else { + mapper.insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); + } + }); + } catch (RuntimeException re) { + if (re.getCause() != null + && re.getCause().getCause() != null + && re.getCause().getCause() instanceof SQLIntegrityConstraintViolationException) { + // TODO We should make more fine-grained exception judgments + // Usually throwing `SQLIntegrityConstraintViolationException` means that + // SQL violates the constraints of `primary key` and `unique key`. + // We simply think that the entity already exists at this time. + throw new EntityAlreadyExistsException( + String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); + } + throw re; } - SessionUtils.doWithCommit( - MetalakeMetaMapper.class, - mapper -> { - if (overwritten) { - mapper.insertMetalakeMetaWithUpdate(POConverters.toMetalakePO((BaseMetalake) e)); - } else { - mapper.insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); - } - }); } else { - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for insert operation", e.getClass())); + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for insert operation", e.getClass()); } } @@ -106,9 +115,9 @@ public E update( BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); Preconditions.checkArgument( Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), - String.format( - "The updated metalake entity id: %s should same with the metalake entity id before: %s", - newMetalakeEntity.id(), oldMetalakeEntity.id())); + "The updated metalake entity id: %s should same with the metalake entity id before: %s", + newMetalakeEntity.id(), + oldMetalakeEntity.id()); Integer updateResult = SessionUtils.doWithCommitAndFetchResult( @@ -122,8 +131,8 @@ public E update( throw new IOException("Failed to update the entity:" + ident); } } else { - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for update operation", entityType)); + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for update operation", entityType); } } @@ -140,8 +149,8 @@ public E get( } return (E) POConverters.fromMetalakePO(metalakePO); } else { - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for get operation", entityType)); + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for get operation", entityType); } } @@ -162,14 +171,16 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea // TODO We will cascade delete the metadata of sub-resources under the metalake }); } else { + // TODO Check whether the sub-resources are empty. If the sub-resources are not empty, + // deletion is not allowed. SessionUtils.doWithCommit( MetalakeMetaMapper.class, mapper -> mapper.deleteMetalakeMetaById(metalakeId)); } } return true; } else { - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for delete operation", entityType)); + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for delete operation", entityType); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index 104c83469fc..155907b1580 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -75,7 +75,7 @@ public interface MetalakeMetaMapper { + " properties = #{metalakeMeta.properties}," + " audit_info = #{metalakeMeta.auditInfo}," + " schema_version = #{metalakeMeta.schemaVersion}") - void insertMetalakeMetaWithUpdate(@Param("metalakeMeta") MetalakePO metalakePO); + void insertMetalakeMetaOnDuplicateKeyUpdate(@Param("metalakeMeta") MetalakePO metalakePO); @Update( "UPDATE " diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index 808ac5a1c3a..3d2f341bc5b 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -22,6 +22,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.Namespace; @@ -53,8 +54,11 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestRelationalEntityStore { + private static final Logger Logger = LoggerFactory.getLogger(TestRelationalEntityStore.class); private static final String MYSQL_STORE_PATH = "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; @@ -114,9 +118,14 @@ public void destroy() { } @AfterAll - public static void tearDown() throws IOException { + public static void tearDown() { dropAllTables(); - entityStore.close(); + try { + entityStore.close(); + } catch (IOException e) { + Logger.error("Close the entity store failed:", e); + } + File dir = new File(DB_DIR); if (dir.exists()) { dir.delete(); @@ -134,7 +143,8 @@ public void testPutAndGet() throws IOException { // overwrite false BaseMetalake duplicateMetalake = createMetalake(1L, "test_metalake", "this is test"); - assertThrows(RuntimeException.class, () -> entityStore.put(duplicateMetalake, false)); + assertThrows( + EntityAlreadyExistsException.class, () -> entityStore.put(duplicateMetalake, false)); // overwrite true BaseMetalake overittenMetalake = createMetalake(1L, "test_metalake2", "this is test2"); From dd806c657c18c9ffdeef602500b1d30c7b86d266 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 19 Feb 2024 12:59:00 +0800 Subject: [PATCH 16/34] remove type config --- .../com/datastrato/gravitino/Configs.java | 11 ----- .../storage/relational/JDBCBackend.java | 2 +- .../session/SqlSessionFactoryHelper.java | 13 +----- .../relational/utils/SessionUtils.java | 46 +++++++++++++++++++ .../relational/TestRelationalEntityStore.java | 4 -- .../relational/session/TestSqlSession.java | 10 ++-- docs/gravitino-server-config.md | 27 ++++++----- 7 files changed, 65 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 48bbb3087f7..384f47628bf 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -22,9 +22,6 @@ public interface Configs { String DEFAULT_ENTITY_RELATIONAL_STORE = "JDBCBackend"; String ENTITY_RELATIONAL_STORE_KEY = "gravitino.entity.store.relational"; - String ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE_KEY = - "gravitino.entity.store.relational.jdbcDBType"; - String DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE = "mysql"; String ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY = "gravitino.entity.store.relational.jdbcUrl"; String ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY = "gravitino.entity.store.relational.jdbcDriver"; String ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY = "gravitino.entity.store.relational.jdbcUser"; @@ -68,14 +65,6 @@ public interface Configs { .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .createWithDefault(DEFAULT_ENTITY_RELATIONAL_STORE); - ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE = - new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE_KEY) - .doc("Database type of `JDBCBackend`") - .version(ConfigConstants.VERSION_0_5_0) - .stringConf() - .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) - .createWithDefault(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); - ConfigEntry ENTITY_RELATIONAL_JDBC_BACKEND_URL = new ConfigBuilder(ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY) .doc("Connection URL of `JDBCBackend`") diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index d50787ad273..483d9e22417 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -31,7 +31,7 @@ * {@link JDBCBackend} is a jdbc implementation of {@link RelationalBackend} interface. You can use * a database that supports the JDBC protocol as storage. If the specified database has special SQL * syntax, please implement the SQL statements and methods in MyBatis Mapper separately and switch - * according to the {@link Configs#ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE_KEY} parameter. + * according to the {@link Configs#ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY} parameter. */ public class JDBCBackend implements RelationalBackend { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index 8a9ad4581bc..aaff7ce6130 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -8,7 +8,6 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.sql.SQLException; import java.time.Duration; @@ -36,11 +35,6 @@ public static SqlSessionFactoryHelper getInstance() { private SqlSessionFactoryHelper() {} - @VisibleForTesting - static void setSqlSessionFactory(SqlSessionFactory sessionFactory) { - sqlSessionFactory = sessionFactory; - } - /** * Initialize the SqlSessionFactory object. * @@ -52,11 +46,8 @@ public void init(Config config) { BasicDataSource dataSource = new BasicDataSource(); dataSource.setUrl(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL)); dataSource.setDriverClassName(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)); - String dbType = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); - if (dbType.equals(Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE)) { - dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER)); - dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)); - } + dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER)); + dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)); // Close the auto commit, so that we can control the transaction manual commit dataSource.setDefaultAutoCommit(false); dataSource.setMaxWaitMillis(1000L); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java index 62ef8243afd..2d9009acfa8 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java @@ -11,9 +11,21 @@ import java.util.function.Function; import org.apache.ibatis.session.SqlSession; +/** + * This class provides utility methods to perform database operations with MyBatis mappers in the + * SqlSession. + */ public class SessionUtils { private SessionUtils() {} + /** + * This method is used to perform a database operation with a commit. If the operation fails, the + * transaction will roll back. + * + * @param mapperClazz mapper class to be used for the operation + * @param consumer the operation to be performed with the mapper + * @param the type of the mapper + */ public static void doWithCommit(Class mapperClazz, Consumer consumer) { try (SqlSession session = SqlSessions.getSqlSession()) { try { @@ -27,6 +39,16 @@ public static void doWithCommit(Class mapperClazz, Consumer consumer) } } + /** + * This method is used to perform a database operation with a commit and fetch the result. If the + * operation fails, the transaction will roll back. + * + * @param mapperClazz mapper class to be used for the operation + * @param func the operation to be performed with the mapper + * @return the result of the operation + * @param the type of the mapper + * @param the type of the result + */ public static R doWithCommitAndFetchResult(Class mapperClazz, Function func) { try (SqlSession session = SqlSessions.getSqlSession()) { try { @@ -41,6 +63,14 @@ public static R doWithCommitAndFetchResult(Class mapperClazz, Function } } + /** + * This method is used to perform a database operation without a commit. If the operation fails, + * will throw the RuntimeException. + * + * @param mapperClazz mapper class to be used for the operation + * @param consumer the operation to be performed with the mapper + * @param the type of the mapper + */ public static void doWithoutCommit(Class mapperClazz, Consumer consumer) { try { T mapper = SqlSessions.getMapper(mapperClazz); @@ -50,6 +80,16 @@ public static void doWithoutCommit(Class mapperClazz, Consumer consume } } + /** + * This method is used to perform a database operation without a commit and fetch the result. If + * the operation fails, will throw a RuntimeException. + * + * @param mapperClazz mapper class to be used for the operation + * @param func the operation to be performed with the mapper + * @return the result of the operation + * @param the type of the mapper + * @param the type of the result + */ public static R getWithoutCommit(Class mapperClazz, Function func) { try (SqlSession session = SqlSessions.getSqlSession()) { try { @@ -63,6 +103,12 @@ public static R getWithoutCommit(Class mapperClazz, Function fun } } + /** + * This method is used to perform multiple database operations with a commit. If any of the + * operations fail, the transaction will totally roll back. + * + * @param operations the operations to be performed + */ public static void doMultipleWithCommit(Runnable... operations) { try (SqlSession session = SqlSessions.getSqlSession()) { try { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index 3d2f341bc5b..e1703c360ff 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -5,9 +5,7 @@ package com.datastrato.gravitino.storage.relational; -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; @@ -76,8 +74,6 @@ public static void setUp() { Config config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE)) - .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java index b6af9553a23..29df8d5afd3 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java @@ -5,9 +5,7 @@ package com.datastrato.gravitino.storage.relational.session; -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; @@ -52,8 +50,6 @@ public static void setUp() { config = Mockito.mock(Config.class); Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE)) - .thenReturn(DEFAULT_ENTITY_RELATIONAL_JDBC_BACKEND_DB_TYPE); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); Mockito.when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("root"); @@ -77,7 +73,7 @@ public static void tearDown() throws IOException { if (dir.exists()) { dir.delete(); } - SqlSessionFactoryHelper.setSqlSessionFactory(null); + SqlSessionFactoryHelper.getInstance().close(); } @Test @@ -88,7 +84,7 @@ public void testGetInstance() { @Test public void testInit() throws SQLException { - SqlSessionFactoryHelper.setSqlSessionFactory(null); + SqlSessionFactoryHelper.getInstance().close(); SqlSessionFactoryHelper.getInstance().init(config); assertNotNull(SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); BasicDataSource dataSource = @@ -104,7 +100,7 @@ public void testInit() throws SQLException { @Test public void testGetSqlSessionFactoryWithoutInit() { - SqlSessionFactoryHelper.setSqlSessionFactory(null); + SqlSessionFactoryHelper.getInstance().close(); assertThrows( IllegalStateException.class, () -> SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 93c2a0caa50..1f8c7618f7e 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -43,20 +43,19 @@ You can also specify filter parameters by setting configuration entries of the f ### Storage configuration -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|--------------------------------------------|---------------| -| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | -| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | -| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | -| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | -| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | -| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | -| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbcDBType` | The database type that the `JDBCBackend` needs to connect to. If you use `MySQL`, you should firstly initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | `mysql` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | -| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` | 0.5.0 | -| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL` storage. | (none) | Yes if you use `JdbcBackend` and `MySQL` | 0.5.0 | +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|----------------------------------|---------------| +| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | +| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | +| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | +| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | +| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | +| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | +| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. If you use `MySQL`, you should firstly initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL`. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL`. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | :::caution We strongly recommend that you change the default value of `gravitino.entity.store.kv.rocksdbPath`, as it's under the deployment directory and future version upgrades may remove it. From fdffa5c3cef1d098a232b2da0ef16d303c981e0e Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Thu, 22 Feb 2024 11:03:08 +0800 Subject: [PATCH 17/34] refactor the schema --- .../storage/relational/JDBCBackend.java | 23 ++++-- .../relational/mapper/MetalakeMetaMapper.java | 67 +++++++++++----- .../storage/relational/po/MetalakePO.java | 76 +++++++++++-------- .../relational/utils/POConverters.java | 45 ++++++++++- .../relational/utils/SessionUtils.java | 6 +- core/src/main/resources/mysql/mysql_init.sql | 9 ++- .../relational/utils/TestPOConverters.java | 6 +- core/src/test/resources/h2/h2-init.sql | 9 ++- 8 files changed, 166 insertions(+), 75 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 483d9e22417..501102bc4dd 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -76,11 +76,13 @@ public void insert(E e, boolean overwritten) SessionUtils.doWithCommit( MetalakeMetaMapper.class, mapper -> { + MetalakePO po = + POConverters.initializeMetalakePOVersion( + POConverters.toMetalakePO((BaseMetalake) e)); if (overwritten) { - mapper.insertMetalakeMetaOnDuplicateKeyUpdate( - POConverters.toMetalakePO((BaseMetalake) e)); + mapper.insertMetalakeMetaOnDuplicateKeyUpdate(po); } else { - mapper.insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); + mapper.insertMetalakeMeta(po); } }); } catch (RuntimeException re) { @@ -110,6 +112,9 @@ public E update( MetalakePO oldMetalakePO = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); + if (oldMetalakePO == null) { + throw new NoSuchEntityException("No such entity:%s", ident.toString()); + } BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); @@ -118,13 +123,14 @@ public E update( "The updated metalake entity id: %s should same with the metalake entity id before: %s", newMetalakeEntity.id(), oldMetalakeEntity.id()); + MetalakePO newMetalakePO = + POConverters.updateMetalakePOVersion( + oldMetalakePO, POConverters.toMetalakePO(newMetalakeEntity)); Integer updateResult = SessionUtils.doWithCommitAndFetchResult( MetalakeMetaMapper.class, - mapper -> - mapper.updateMetalakeMeta( - POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO)); + mapper -> mapper.updateMetalakeMeta(newMetalakePO, oldMetalakePO)); if (updateResult > 0) { return (E) newMetalakeEntity; } else { @@ -166,7 +172,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea () -> SessionUtils.doWithoutCommit( MetalakeMetaMapper.class, - mapper -> mapper.deleteMetalakeMetaById(metalakeId)), + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)), () -> { // TODO We will cascade delete the metadata of sub-resources under the metalake }); @@ -174,7 +180,8 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea // TODO Check whether the sub-resources are empty. If the sub-resources are not empty, // deletion is not allowed. SessionUtils.doWithCommit( - MetalakeMetaMapper.class, mapper -> mapper.deleteMetalakeMetaById(metalakeId)); + MetalakeMetaMapper.class, + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)); } } return true; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index 155907b1580..b731b718442 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -7,7 +7,6 @@ import com.datastrato.gravitino.storage.relational.po.MetalakePO; import java.util.List; -import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -25,56 +24,76 @@ public interface MetalakeMetaMapper { String TABLE_NAME = "metalake_meta"; @Select( - "SELECT id, metalake_name as metalakeName, metalake_comment as metalakeComment," - + " properties, audit_info as auditInfo, schema_version as schemaVersion" + "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + " FROM " - + TABLE_NAME) + + TABLE_NAME + + " WHERE deleted_at = 0") List listMetalakePOs(); @Select( - "SELECT id, metalake_name as metalakeName," + "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion" + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + " FROM " + TABLE_NAME - + " WHERE metalake_name = #{metalakeName}") + + " WHERE metalake_name = #{metalakeName} and deleted_at = 0") MetalakePO selectMetalakeMetaByName(@Param("metalakeName") String name); - @Select("SELECT id FROM " + TABLE_NAME + " WHERE metalake_name = #{metalakeName}") + @Select( + "SELECT metalake_id FROM " + + TABLE_NAME + + " WHERE metalake_name = #{metalakeName} and deleted_at = 0") Long selectMetalakeIdMetaByName(@Param("metalakeName") String name); @Insert( "INSERT INTO " + TABLE_NAME - + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + " VALUES(" - + " #{metalakeMeta.id}," + + " #{metalakeMeta.metalakeId}," + " #{metalakeMeta.metalakeName}," + " #{metalakeMeta.metalakeComment}," + " #{metalakeMeta.properties}," + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}" + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + " )") void insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); @Insert( "INSERT INTO " + TABLE_NAME - + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + " VALUES(" - + " #{metalakeMeta.id}," + + " #{metalakeMeta.metalakeId}," + " #{metalakeMeta.metalakeName}," + " #{metalakeMeta.metalakeComment}," + " #{metalakeMeta.properties}," + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}" + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + " )" + " ON DUPLICATE KEY UPDATE" + " metalake_name = #{metalakeMeta.metalakeName}," + " metalake_comment = #{metalakeMeta.metalakeComment}," + " properties = #{metalakeMeta.properties}," + " audit_info = #{metalakeMeta.auditInfo}," - + " schema_version = #{metalakeMeta.schemaVersion}") + + " schema_version = #{metalakeMeta.schemaVersion}," + + " current_version = #{metalakeMeta.currentVersion}," + + " last_version = #{metalakeMeta.lastVersion}," + + " deleted_at = #{metalakeMeta.deletedAt}") void insertMetalakeMetaOnDuplicateKeyUpdate(@Param("metalakeMeta") MetalakePO metalakePO); @Update( @@ -84,17 +103,25 @@ public interface MetalakeMetaMapper { + " metalake_comment = #{newMetalakeMeta.metalakeComment}," + " properties = #{newMetalakeMeta.properties}," + " audit_info = #{newMetalakeMeta.auditInfo}," - + " schema_version = #{newMetalakeMeta.schemaVersion}" - + " WHERE id = #{oldMetalakeMeta.id}" + + " schema_version = #{newMetalakeMeta.schemaVersion}," + + " current_version = #{newMetalakeMeta.currentVersion}," + + " last_version = #{newMetalakeMeta.lastVersion}" + + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" + " and metalake_name = #{oldMetalakeMeta.metalakeName}" + " and metalake_comment = #{oldMetalakeMeta.metalakeComment}" + " and properties = #{oldMetalakeMeta.properties}" + " and audit_info = #{oldMetalakeMeta.auditInfo}" - + " and schema_version = #{oldMetalakeMeta.schemaVersion}") + + " and schema_version = #{oldMetalakeMeta.schemaVersion}" + + " and current_version = #{oldMetalakeMeta.currentVersion}" + + " and last_version = #{oldMetalakeMeta.lastVersion}" + + " and deleted_at = 0") Integer updateMetalakeMeta( @Param("newMetalakeMeta") MetalakePO newMetalakePO, @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); - @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") - Integer deleteMetalakeMetaById(@Param("id") Long id); + @Update( + "UPDATE " + + TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP() WHERE metalake_id = #{metalakeId}") + Integer softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/MetalakePO.java index 9d833bb4861..61182f10fb2 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/MetalakePO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/MetalakePO.java @@ -8,59 +8,50 @@ import com.google.common.base.Objects; public class MetalakePO { - private Long id; + private Long metalakeId; private String metalakeName; private String metalakeComment; private String properties; private String auditInfo; private String schemaVersion; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; + public Long getMetalakeId() { + return metalakeId; } public String getMetalakeName() { return metalakeName; } - public void setMetalakeName(String metalakeName) { - this.metalakeName = metalakeName; - } - public String getMetalakeComment() { return metalakeComment; } - public void setMetalakeComment(String metalakeComment) { - this.metalakeComment = metalakeComment; - } - public String getProperties() { return properties; } - public void setProperties(String properties) { - this.properties = properties; - } - public String getAuditInfo() { return auditInfo; } - public void setAuditInfo(String auditInfo) { - this.auditInfo = auditInfo; - } - public String getSchemaVersion() { return schemaVersion; } - public void setSchemaVersion(String schemaVersion) { - this.schemaVersion = schemaVersion; + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; } @Override @@ -72,23 +63,29 @@ public boolean equals(Object o) { return false; } MetalakePO that = (MetalakePO) o; - return Objects.equal(getId(), that.getId()) + return Objects.equal(getMetalakeId(), that.getMetalakeId()) && Objects.equal(getMetalakeName(), that.getMetalakeName()) && Objects.equal(getMetalakeComment(), that.getMetalakeComment()) && Objects.equal(getProperties(), that.getProperties()) && Objects.equal(getAuditInfo(), that.getAuditInfo()) - && Objects.equal(getSchemaVersion(), that.getSchemaVersion()); + && Objects.equal(getSchemaVersion(), that.getSchemaVersion()) + && Objects.equal(getCurrentVersion(), that.getCurrentVersion()) + && Objects.equal(getLastVersion(), that.getLastVersion()) + && Objects.equal(getDeletedAt(), that.getDeletedAt()); } @Override public int hashCode() { return Objects.hashCode( - getId(), + getMetalakeId(), getMetalakeName(), getMetalakeComment(), getProperties(), getAuditInfo(), - getSchemaVersion()); + getSchemaVersion(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); } public static class Builder { @@ -98,8 +95,8 @@ public Builder() { metalakePO = new MetalakePO(); } - public MetalakePO.Builder withId(Long id) { - metalakePO.id = id; + public MetalakePO.Builder withMetalakeId(Long id) { + metalakePO.metalakeId = id; return this; } @@ -123,11 +120,26 @@ public MetalakePO.Builder withAuditInfo(String auditInfo) { return this; } - public MetalakePO.Builder withVersion(String version) { + public MetalakePO.Builder withSchemaVersion(String version) { metalakePO.schemaVersion = version; return this; } + public MetalakePO.Builder withCurrentVersion(Long currentVersion) { + metalakePO.currentVersion = currentVersion; + return this; + } + + public MetalakePO.Builder withLastVersion(Long lastVersion) { + metalakePO.lastVersion = lastVersion; + return this; + } + + public MetalakePO.Builder withDeletedAt(Long deletedAt) { + metalakePO.deletedAt = deletedAt; + return this; + } + public MetalakePO build() { return metalakePO; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 1dbf3cbf7cc..9510408a932 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -29,18 +29,57 @@ private POConverters() {} public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { try { return new MetalakePO.Builder() - .withId(baseMetalake.id()) + .withMetalakeId(baseMetalake.id()) .withMetalakeName(baseMetalake.name()) .withMetalakeComment(baseMetalake.comment()) .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.properties())) .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.auditInfo())) - .withVersion(JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.getVersion())) + .withSchemaVersion( + JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.getVersion())) .build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to serialize json object:", e); } } + /** + * Initialize MetalakePO version + * + * @param metalakePO MetalakePO object + * @return MetalakePO object with version initialized + */ + public static MetalakePO initializeMetalakePOVersion(MetalakePO metalakePO) { + return new MetalakePO.Builder() + .withMetalakeId(metalakePO.getMetalakeId()) + .withMetalakeName(metalakePO.getMetalakeName()) + .withMetalakeComment(metalakePO.getMetalakeComment()) + .withProperties(metalakePO.getProperties()) + .withAuditInfo(metalakePO.getAuditInfo()) + .withSchemaVersion(metalakePO.getSchemaVersion()) + .withCurrentVersion(1L) + .withLastVersion(1L) + .withDeletedAt(0L) + .build(); + } + + public static MetalakePO updateMetalakePOVersion( + MetalakePO oldMetalakePO, MetalakePO newMetalakePO) { + Long lastVersion = oldMetalakePO.getLastVersion(); + // Will set the version to the last version + 1 when having some fields need be multiple version + Long nextVersion = lastVersion; + return new MetalakePO.Builder() + .withMetalakeId(newMetalakePO.getMetalakeId()) + .withMetalakeName(newMetalakePO.getMetalakeName()) + .withMetalakeComment(newMetalakePO.getMetalakeComment()) + .withProperties(newMetalakePO.getProperties()) + .withAuditInfo(newMetalakePO.getAuditInfo()) + .withSchemaVersion(newMetalakePO.getSchemaVersion()) + .withCurrentVersion(nextVersion) + .withLastVersion(nextVersion) + .withDeletedAt(0L) + .build(); + } + /** * Convert {@link MetalakePO} to {@link BaseMetalake} * @@ -50,7 +89,7 @@ public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) { try { return new BaseMetalake.Builder() - .withId(metalakePO.getId()) + .withId(metalakePO.getMetalakeId()) .withName(metalakePO.getMetalakeName()) .withComment(metalakePO.getMetalakeComment()) .withProperties( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java index 2d9009acfa8..ef4db551a2f 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java @@ -29,7 +29,7 @@ private SessionUtils() {} public static void doWithCommit(Class mapperClazz, Consumer consumer) { try (SqlSession session = SqlSessions.getSqlSession()) { try { - T mapper = SqlSessions.getMapper(mapperClazz); + T mapper = session.getMapper(mapperClazz); consumer.accept(mapper); SqlSessions.commitAndCloseSqlSession(); } catch (Throwable t) { @@ -52,7 +52,7 @@ public static void doWithCommit(Class mapperClazz, Consumer consumer) public static R doWithCommitAndFetchResult(Class mapperClazz, Function func) { try (SqlSession session = SqlSessions.getSqlSession()) { try { - T mapper = SqlSessions.getMapper(mapperClazz); + T mapper = session.getMapper(mapperClazz); R result = func.apply(mapper); SqlSessions.commitAndCloseSqlSession(); return result; @@ -93,7 +93,7 @@ public static void doWithoutCommit(Class mapperClazz, Consumer consume public static R getWithoutCommit(Class mapperClazz, Function func) { try (SqlSession session = SqlSessions.getSqlSession()) { try { - T mapper = SqlSessions.getMapper(mapperClazz); + T mapper = session.getMapper(mapperClazz); return func.apply(mapper); } catch (Throwable t) { throw new RuntimeException(t); diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql index bb068a416bc..7dd0fc32224 100644 --- a/core/src/main/resources/mysql/mysql_init.sql +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -4,12 +4,15 @@ -- CREATE TABLE IF NOT EXISTS `metalake_meta` ( - `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', - `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', - PRIMARY KEY (`id`), + `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', + PRIMARY KEY (`metalake_id`), UNIQUE KEY `uk_mn` (`metalake_name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java index fc10c48dbcb..1723f75a1c0 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java @@ -52,7 +52,7 @@ public void testToMetalakePO() throws JsonProcessingException { MetalakePO actualMetalakePO = POConverters.toMetalakePO(metalake); // Assert - assertEquals(expectedMetalakePO.getId(), actualMetalakePO.getId()); + assertEquals(expectedMetalakePO.getMetalakeId(), actualMetalakePO.getMetalakeId()); assertEquals(expectedMetalakePO.getMetalakeName(), actualMetalakePO.getMetalakeName()); assertEquals(expectedMetalakePO.getMetalakeComment(), actualMetalakePO.getMetalakeComment()); assertEquals(expectedMetalakePO.getProperties(), actualMetalakePO.getProperties()); @@ -82,12 +82,12 @@ private static MetalakePO createMetalakePO(Long id, String name, String comment) Map properties = new HashMap<>(); properties.put("key", "value"); return new MetalakePO.Builder() - .withId(id) + .withMetalakeId(id) .withMetalakeName(name) .withMetalakeComment(comment) .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(properties)) .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(auditInfo)) - .withVersion(JsonUtils.anyFieldMapper().writeValueAsString(SchemaVersion.V_0_1)) + .withSchemaVersion(JsonUtils.anyFieldMapper().writeValueAsString(SchemaVersion.V_0_1)) .build(); } } diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index af83ecdf197..5c1f3af998a 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -4,12 +4,15 @@ -- CREATE TABLE IF NOT EXISTS `metalake_meta` ( - `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', - `schema_version` TEXT NOT NULL COMMENT 'metalake schema version info', - PRIMARY KEY (id), + `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', + PRIMARY KEY (metalake_id), CONSTRAINT uk_mn UNIQUE (metalake_name) ) ENGINE = InnoDB; \ No newline at end of file From 4130dce9eca5f2b02b6886945793891639e44853 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Thu, 22 Feb 2024 22:01:27 +0800 Subject: [PATCH 18/34] fix the import order --- core/build.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 25628e6206b..6f6fb3b7266 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -16,16 +16,16 @@ dependencies { implementation(libs.bundles.metrics) implementation(libs.bundles.prometheus) implementation(libs.caffeine) + implementation(libs.commons.dbcp2) implementation(libs.commons.io) implementation(libs.commons.lang3) implementation(libs.guava) + implementation(libs.mybatis) implementation(libs.protobuf.java.util) { exclude("com.google.guava", "guava") .because("Brings in Guava for Android, which we don't want (and breaks multimaps).") } implementation(libs.rocksdbjni) - implementation(libs.commons.dbcp2) - implementation(libs.mybatis) annotationProcessor(libs.lombok) compileOnly(libs.lombok) @@ -33,10 +33,10 @@ dependencies { testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) + testImplementation(libs.h2db) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mockito.core) testRuntimeOnly(libs.junit.jupiter.engine) - testImplementation(libs.h2db) } From e5c25f2550bc96e41a1ad91f332a5e387e8d609c Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Thu, 22 Feb 2024 22:13:44 +0800 Subject: [PATCH 19/34] update --- .../relational/session/SqlSessions.java | 25 +++++++--- .../relational/utils/POConverters.java | 7 +++ .../relational/utils/SessionUtils.java | 6 +-- .../relational/utils/TestPOConverters.java | 49 +++++++++++++++++++ 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java index ca3684981a0..e9d9128003a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessions.java @@ -50,9 +50,12 @@ public static SqlSession getSqlSession() { public static void commitAndCloseSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { - sqlSession.commit(); - sqlSession.close(); - sessions.remove(); + try { + sqlSession.commit(); + sqlSession.close(); + } finally { + sessions.remove(); + } } } @@ -63,9 +66,12 @@ public static void commitAndCloseSqlSession() { public static void rollbackAndCloseSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { - sqlSession.rollback(); - sqlSession.close(); - sessions.remove(); + try { + sqlSession.rollback(); + sqlSession.close(); + } finally { + sessions.remove(); + } } } @@ -73,8 +79,11 @@ public static void rollbackAndCloseSqlSession() { public static void closeSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { - sqlSession.close(); - sessions.remove(); + try { + sqlSession.close(); + } finally { + sessions.remove(); + } } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 9510408a932..679f143f1f5 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -62,6 +62,13 @@ public static MetalakePO initializeMetalakePOVersion(MetalakePO metalakePO) { .build(); } + /** + * Update MetalakePO version + * + * @param oldMetalakePO the old MetalakePO object + * @param newMetalakePO the new MetalakePO object + * @return MetalakePO object with updated version + */ public static MetalakePO updateMetalakePOVersion( MetalakePO oldMetalakePO, MetalakePO newMetalakePO) { Long lastVersion = oldMetalakePO.getLastVersion(); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java index ef4db551a2f..2d9009acfa8 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java @@ -29,7 +29,7 @@ private SessionUtils() {} public static void doWithCommit(Class mapperClazz, Consumer consumer) { try (SqlSession session = SqlSessions.getSqlSession()) { try { - T mapper = session.getMapper(mapperClazz); + T mapper = SqlSessions.getMapper(mapperClazz); consumer.accept(mapper); SqlSessions.commitAndCloseSqlSession(); } catch (Throwable t) { @@ -52,7 +52,7 @@ public static void doWithCommit(Class mapperClazz, Consumer consumer) public static R doWithCommitAndFetchResult(Class mapperClazz, Function func) { try (SqlSession session = SqlSessions.getSqlSession()) { try { - T mapper = session.getMapper(mapperClazz); + T mapper = SqlSessions.getMapper(mapperClazz); R result = func.apply(mapper); SqlSessions.commitAndCloseSqlSession(); return result; @@ -93,7 +93,7 @@ public static void doWithoutCommit(Class mapperClazz, Consumer consume public static R getWithoutCommit(Class mapperClazz, Function func) { try (SqlSession session = SqlSessions.getSqlSession()) { try { - T mapper = session.getMapper(mapperClazz); + T mapper = SqlSessions.getMapper(mapperClazz); return func.apply(mapper); } catch (Throwable t) { throw new RuntimeException(t); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java index 1723f75a1c0..736bbd91595 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java @@ -16,7 +16,10 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; @@ -43,6 +46,52 @@ public void testFromMetalakePO() throws JsonProcessingException { assertEquals(expectedMetalake.getVersion(), convertedMetalake.getVersion()); } + @Test + public void testFromMetalakePOs() throws JsonProcessingException { + MetalakePO metalakePO1 = createMetalakePO(1L, "test", "this is test"); + MetalakePO metalakePO2 = createMetalakePO(2L, "test2", "this is test2"); + List metalakePOs = new ArrayList<>(Arrays.asList(metalakePO1, metalakePO2)); + List convertedMetalakes = POConverters.fromMetalakePOs(metalakePOs); + + BaseMetalake expectedMetalake1 = createMetalake(1L, "test", "this is test"); + BaseMetalake expectedMetalake2 = createMetalake(2L, "test2", "this is test2"); + List expectedMetalakes = + new ArrayList<>(Arrays.asList(expectedMetalake1, expectedMetalake2)); + + // Assert + int index = 0; + for (BaseMetalake metalake : convertedMetalakes) { + assertEquals(expectedMetalakes.get(index).id(), metalake.id()); + assertEquals(expectedMetalakes.get(index).name(), metalake.name()); + assertEquals(expectedMetalakes.get(index).comment(), metalake.comment()); + assertEquals( + expectedMetalakes.get(index).properties().get("key"), metalake.properties().get("key")); + assertEquals( + expectedMetalakes.get(index).auditInfo().creator(), metalake.auditInfo().creator()); + assertEquals(expectedMetalakes.get(index).getVersion(), metalake.getVersion()); + index++; + } + } + + @Test + public void testInitMetalakePOVersion() throws JsonProcessingException { + MetalakePO metalakePO = createMetalakePO(1L, "test", "this is test"); + MetalakePO initPO = POConverters.initializeMetalakePOVersion(metalakePO); + assertEquals(1, initPO.getCurrentVersion()); + assertEquals(1, initPO.getLastVersion()); + assertEquals(0, initPO.getDeletedAt()); + } + + @Test + public void testUpdateMetalakePOVersion() throws JsonProcessingException { + MetalakePO metalakePO = createMetalakePO(1L, "test", "this is test"); + MetalakePO initPO = POConverters.initializeMetalakePOVersion(metalakePO); + MetalakePO updatePO = POConverters.updateMetalakePOVersion(initPO, initPO); + assertEquals(1, initPO.getCurrentVersion()); + assertEquals(1, initPO.getLastVersion()); + assertEquals(0, initPO.getDeletedAt()); + } + @Test public void testToMetalakePO() throws JsonProcessingException { BaseMetalake metalake = createMetalake(1L, "test", "this is test"); From 7760554348529309e41b373fccdc0bdd2e104659 Mon Sep 17 00:00:00 2001 From: Jiebao Xiao Date: Sun, 25 Feb 2024 14:51:54 +0800 Subject: [PATCH 20/34] fix comments --- .../UnsupportedEntityTypeException.java | 3 +- .../storage/relational/JDBCBackend.java | 154 ++++-------------- .../service/MetalakeMetaService.java | 133 +++++++++++++++ .../relational/utils/POConverters.java | 18 +- core/src/main/resources/mysql/mysql_init.sql | 2 +- .../relational/utils/TestPOConverters.java | 12 +- core/src/test/resources/h2/h2-init.sql | 2 +- docs/gravitino-server-config.md | 26 +-- 8 files changed, 202 insertions(+), 148 deletions(-) rename {api/src/main/java/com/datastrato/gravitino/exceptions => core/src/main/java/com/datastrato/gravitino}/UnsupportedEntityTypeException.java (91%) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/UnsupportedEntityTypeException.java b/core/src/main/java/com/datastrato/gravitino/UnsupportedEntityTypeException.java similarity index 91% rename from api/src/main/java/com/datastrato/gravitino/exceptions/UnsupportedEntityTypeException.java rename to core/src/main/java/com/datastrato/gravitino/UnsupportedEntityTypeException.java index 9e6aaa55d34..8f0eb31fef6 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/UnsupportedEntityTypeException.java +++ b/core/src/main/java/com/datastrato/gravitino/UnsupportedEntityTypeException.java @@ -2,8 +2,9 @@ * Copyright 2024 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.exceptions; +package com.datastrato.gravitino; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.google.errorprone.annotations.FormatMethod; import com.google.errorprone.annotations.FormatString; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 501102bc4dd..d7825445bcd 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -2,6 +2,7 @@ * Copyright 2024 Datastrato Pvt Ltd. * This software is licensed under the Apache License version 2. */ + package com.datastrato.gravitino.storage.relational; import com.datastrato.gravitino.Config; @@ -11,20 +12,14 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.UnsupportedEntityTypeException; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.exceptions.UnsupportedEntityTypeException; import com.datastrato.gravitino.meta.BaseMetalake; -import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; -import com.datastrato.gravitino.storage.relational.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; -import com.datastrato.gravitino.storage.relational.utils.POConverters; -import com.datastrato.gravitino.storage.relational.utils.SessionUtils; -import com.google.common.base.Preconditions; import java.io.IOException; -import java.sql.SQLIntegrityConstraintViolationException; import java.util.List; -import java.util.Objects; import java.util.function.Function; /** @@ -44,27 +39,22 @@ public void initialize(Config config) { @Override public List list( Namespace namespace, Entity.EntityType entityType) { - if (entityType == Entity.EntityType.METALAKE) { - List metalakePOS = - SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, MetalakeMetaMapper::listMetalakePOs); - return (List) POConverters.fromMetalakePOs(metalakePOS); - } else { - throw new UnsupportedEntityTypeException( - "Unsupported entity type: %s for list operation", entityType); + switch (entityType) { + case METALAKE: + return (List) MetalakeMetaService.getInstance().listMetalakes(); + default: + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for list operation", entityType); } } @Override public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { - if (entityType == Entity.EntityType.METALAKE) { - MetalakePO metalakePO = - SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); - return metalakePO != null; - } else { - throw new UnsupportedEntityTypeException( - "Unsupported entity type: %s for exists operation", entityType); + try { + Entity entity = get(ident, entityType); + return entity != null; + } catch (NoSuchEntityException ne) { + return false; } } @@ -72,32 +62,7 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { public void insert(E e, boolean overwritten) throws EntityAlreadyExistsException { if (e instanceof BaseMetalake) { - try { - SessionUtils.doWithCommit( - MetalakeMetaMapper.class, - mapper -> { - MetalakePO po = - POConverters.initializeMetalakePOVersion( - POConverters.toMetalakePO((BaseMetalake) e)); - if (overwritten) { - mapper.insertMetalakeMetaOnDuplicateKeyUpdate(po); - } else { - mapper.insertMetalakeMeta(po); - } - }); - } catch (RuntimeException re) { - if (re.getCause() != null - && re.getCause().getCause() != null - && re.getCause().getCause() instanceof SQLIntegrityConstraintViolationException) { - // TODO We should make more fine-grained exception judgments - // Usually throwing `SQLIntegrityConstraintViolationException` means that - // SQL violates the constraints of `primary key` and `unique key`. - // We simply think that the entity already exists at this time. - throw new EntityAlreadyExistsException( - String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); - } - throw re; - } + MetalakeMetaService.getInstance().insertMetalake((BaseMetalake) e, overwritten); } else { throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for insert operation", e.getClass()); @@ -108,86 +73,35 @@ public void insert(E e, boolean overwritten) public E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) throws IOException, NoSuchEntityException, AlreadyExistsException { - if (entityType == Entity.EntityType.METALAKE) { - MetalakePO oldMetalakePO = - SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); - if (oldMetalakePO == null) { - throw new NoSuchEntityException("No such entity:%s", ident.toString()); - } - - BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); - BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); - Preconditions.checkArgument( - Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), - "The updated metalake entity id: %s should same with the metalake entity id before: %s", - newMetalakeEntity.id(), - oldMetalakeEntity.id()); - MetalakePO newMetalakePO = - POConverters.updateMetalakePOVersion( - oldMetalakePO, POConverters.toMetalakePO(newMetalakeEntity)); - - Integer updateResult = - SessionUtils.doWithCommitAndFetchResult( - MetalakeMetaMapper.class, - mapper -> mapper.updateMetalakeMeta(newMetalakePO, oldMetalakePO)); - if (updateResult > 0) { - return (E) newMetalakeEntity; - } else { - throw new IOException("Failed to update the entity:" + ident); - } - } else { - throw new UnsupportedEntityTypeException( - "Unsupported entity type: %s for update operation", entityType); + switch (entityType) { + case METALAKE: + return (E) MetalakeMetaService.getInstance().updateMetalake(ident, updater); + default: + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for update operation", entityType); } } @Override public E get( - NameIdentifier ident, Entity.EntityType entityType) - throws NoSuchEntityException, IOException { - if (entityType == Entity.EntityType.METALAKE) { - MetalakePO metalakePO = - SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); - if (metalakePO == null) { - throw new NoSuchEntityException("No such entity:%s", ident.toString()); - } - return (E) POConverters.fromMetalakePO(metalakePO); - } else { - throw new UnsupportedEntityTypeException( - "Unsupported entity type: %s for get operation", entityType); + NameIdentifier ident, Entity.EntityType entityType) throws NoSuchEntityException { + switch (entityType) { + case METALAKE: + return (E) MetalakeMetaService.getInstance().getMetalakeByIdent(ident); + default: + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for get operation", entityType); } } @Override public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { - if (entityType == Entity.EntityType.METALAKE) { - Long metalakeId = - SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(ident.name())); - if (metalakeId != null) { - if (cascade) { - SessionUtils.doMultipleWithCommit( - () -> - SessionUtils.doWithoutCommit( - MetalakeMetaMapper.class, - mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)), - () -> { - // TODO We will cascade delete the metadata of sub-resources under the metalake - }); - } else { - // TODO Check whether the sub-resources are empty. If the sub-resources are not empty, - // deletion is not allowed. - SessionUtils.doWithCommit( - MetalakeMetaMapper.class, - mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)); - } - } - return true; - } else { - throw new UnsupportedEntityTypeException( - "Unsupported entity type: %s for delete operation", entityType); + switch (entityType) { + case METALAKE: + return MetalakeMetaService.getInstance().deleteMetalake(ident, cascade); + default: + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for delete operation", entityType); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java new file mode 100644 index 00000000000..9cb1418e444 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -0,0 +1,133 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relational.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.utils.POConverters; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +public class MetalakeMetaService { + private static final MetalakeMetaService INSTANCE = new MetalakeMetaService(); + + public static MetalakeMetaService getInstance() { + return INSTANCE; + } + + private MetalakeMetaService() {} + + public List listMetalakes() { + List metalakePOS = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, MetalakeMetaMapper::listMetalakePOs); + return POConverters.fromMetalakePOs(metalakePOS); + } + + public BaseMetalake getMetalakeByIdent(NameIdentifier ident) { + MetalakePO metalakePO = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); + if (metalakePO == null) { + throw new NoSuchEntityException("No such entity: %s", ident.toString()); + } + return POConverters.fromMetalakePO(metalakePO); + } + + public void insertMetalake(BaseMetalake baseMetalake, boolean overwrite) { + try { + SessionUtils.doWithCommit( + MetalakeMetaMapper.class, + mapper -> { + MetalakePO po = POConverters.initializeMetalakePOWithVersion(baseMetalake); + if (overwrite) { + mapper.insertMetalakeMetaOnDuplicateKeyUpdate(po); + } else { + mapper.insertMetalakeMeta(po); + } + }); + } catch (RuntimeException re) { + if (re.getCause() != null + && re.getCause().getCause() != null + && re.getCause().getCause() instanceof SQLIntegrityConstraintViolationException) { + // TODO We should make more fine-grained exception judgments + // Usually throwing `SQLIntegrityConstraintViolationException` means that + // SQL violates the constraints of `primary key` and `unique key`. + // We simply think that the entity already exists at this time. + throw new EntityAlreadyExistsException( + String.format( + "Metalake entity: %s already exists", baseMetalake.nameIdentifier().name())); + } + throw re; + } + } + + public BaseMetalake updateMetalake( + NameIdentifier ident, Function updater) throws IOException { + MetalakePO oldMetalakePO = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); + if (oldMetalakePO == null) { + throw new NoSuchEntityException("No such entity: %s", ident.toString()); + } + + BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); + BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); + Preconditions.checkArgument( + Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), + "The updated metalake entity id: %s should be same with the metalake entity id before: %s", + newMetalakeEntity.id(), + oldMetalakeEntity.id()); + MetalakePO newMetalakePO = + POConverters.updateMetalakePOWithVersion(oldMetalakePO, newMetalakeEntity); + + Integer updateResult = + SessionUtils.doWithCommitAndFetchResult( + MetalakeMetaMapper.class, + mapper -> mapper.updateMetalakeMeta(newMetalakePO, oldMetalakePO)); + if (updateResult > 0) { + return newMetalakeEntity; + } else { + throw new IOException("Failed to update the entity: " + ident); + } + } + + public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(ident.name())); + if (metalakeId != null) { + if (cascade) { + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + MetalakeMetaMapper.class, + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)), + () -> { + // TODO We will cascade delete the metadata of sub-resources under the metalake + }); + } else { + // TODO Check whether the sub-resources are empty. If the sub-resources are not empty, + // deletion is not allowed. + SessionUtils.doWithCommit( + MetalakeMetaMapper.class, + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)); + } + } + return true; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 679f143f1f5..0c03faa77ba 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -11,6 +11,7 @@ import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.annotations.VisibleForTesting; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -26,7 +27,8 @@ private POConverters() {} * @param baseMetalake BaseMetalake object * @return MetalakePO object from BaseMetalake object */ - public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { + @VisibleForTesting + static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { try { return new MetalakePO.Builder() .withMetalakeId(baseMetalake.id()) @@ -43,12 +45,13 @@ public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { } /** - * Initialize MetalakePO version + * Initialize MetalakePO * - * @param metalakePO MetalakePO object + * @param baseMetalake BaseMetalake object * @return MetalakePO object with version initialized */ - public static MetalakePO initializeMetalakePOVersion(MetalakePO metalakePO) { + public static MetalakePO initializeMetalakePOWithVersion(BaseMetalake baseMetalake) { + MetalakePO metalakePO = toMetalakePO(baseMetalake); return new MetalakePO.Builder() .withMetalakeId(metalakePO.getMetalakeId()) .withMetalakeName(metalakePO.getMetalakeName()) @@ -66,11 +69,12 @@ public static MetalakePO initializeMetalakePOVersion(MetalakePO metalakePO) { * Update MetalakePO version * * @param oldMetalakePO the old MetalakePO object - * @param newMetalakePO the new MetalakePO object + * @param newMetalake the new BaseMetalake object * @return MetalakePO object with updated version */ - public static MetalakePO updateMetalakePOVersion( - MetalakePO oldMetalakePO, MetalakePO newMetalakePO) { + public static MetalakePO updateMetalakePOWithVersion( + MetalakePO oldMetalakePO, BaseMetalake newMetalake) { + MetalakePO newMetalakePO = toMetalakePO(newMetalake); Long lastVersion = oldMetalakePO.getLastVersion(); // Will set the version to the last version + 1 when having some fields need be multiple version Long nextVersion = lastVersion; diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql index 7dd0fc32224..77f300810b6 100644 --- a/core/src/main/resources/mysql/mysql_init.sql +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -14,5 +14,5 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', PRIMARY KEY (`metalake_id`), - UNIQUE KEY `uk_mn` (`metalake_name`) + UNIQUE KEY `uk_mn_del` (`metalake_name`, `deleted_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java index 736bbd91595..285e756c578 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java @@ -75,8 +75,8 @@ public void testFromMetalakePOs() throws JsonProcessingException { @Test public void testInitMetalakePOVersion() throws JsonProcessingException { - MetalakePO metalakePO = createMetalakePO(1L, "test", "this is test"); - MetalakePO initPO = POConverters.initializeMetalakePOVersion(metalakePO); + BaseMetalake metalakePO = createMetalake(1L, "test", "this is test"); + MetalakePO initPO = POConverters.initializeMetalakePOWithVersion(metalakePO); assertEquals(1, initPO.getCurrentVersion()); assertEquals(1, initPO.getLastVersion()); assertEquals(0, initPO.getDeletedAt()); @@ -84,12 +84,14 @@ public void testInitMetalakePOVersion() throws JsonProcessingException { @Test public void testUpdateMetalakePOVersion() throws JsonProcessingException { - MetalakePO metalakePO = createMetalakePO(1L, "test", "this is test"); - MetalakePO initPO = POConverters.initializeMetalakePOVersion(metalakePO); - MetalakePO updatePO = POConverters.updateMetalakePOVersion(initPO, initPO); + BaseMetalake metalake = createMetalake(1L, "test", "this is test"); + BaseMetalake updatedMetalake = createMetalake(1L, "test", "this is test2"); + MetalakePO initPO = POConverters.initializeMetalakePOWithVersion(metalake); + MetalakePO updatePO = POConverters.updateMetalakePOWithVersion(initPO, updatedMetalake); assertEquals(1, initPO.getCurrentVersion()); assertEquals(1, initPO.getLastVersion()); assertEquals(0, initPO.getDeletedAt()); + assertEquals("this is test2", updatePO.getMetalakeComment()); } @Test diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index 5c1f3af998a..74d00874577 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -14,5 +14,5 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', PRIMARY KEY (metalake_id), - CONSTRAINT uk_mn UNIQUE (metalake_name) + CONSTRAINT uk_mn_del UNIQUE (metalake_name, deleted_at) ) ENGINE = InnoDB; \ No newline at end of file diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 1f8c7618f7e..f70259436cf 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -43,19 +43,19 @@ You can also specify filter parameters by setting configuration entries of the f ### Storage configuration -| Configuration item | Description | Default value | Required | Since version | -|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|----------------------------------|---------------| -| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage is currently supported, and the default value is `kv`. | `kv` | No | 0.1.0 | -| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | -| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | -| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | -| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | -| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | -| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. If you use `MySQL`, you should firstly initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | -| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | -| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL`. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | -| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL`. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| Configuration item | Description | Default value | Required | Since version | +|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|----------------------------------|---------------| +| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage and relational storage are currently supported, the default value is `kv`, and the optional value is `relational`. | `kv` | No | 0.1.0 | +| `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | +| `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | +| `graivitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | +| `gravitino.entity.store.maxTransactionSkewTimeMs` | The maximum skew time of transactions in milliseconds. | `2000` | No | 0.3.0 | +| `gravitino.entity.store.kv.deleteAfterTimeMs` | The maximum time in milliseconds that deleted and old-version data is kept. Set to at least 10 minutes and no longer than 30 days. | `604800000`(7 days) | No | 0.3.0 | +| `gravitino.entity.store.relational` | Detailed implementation of Relational storage. `MySQL` is currently supported, and the implementation is `JDBCBackend`. | `JDBCBackend` | No | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUrl` | The database url that the `JDBCBackend` needs to connect to. If you use `MySQL`, you should firstly initialize the database tables yourself by executing the file named `mysql_init.sql` in the `src/main/resources/mysql` directory of the `core` module. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcDriver` | The jdbc driver name that the `JDBCBackend` needs to use. You should place the driver Jar package in the `${GRAVITINO_HOME}/libs/` directory. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcUser` | The username that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL`. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | +| `gravitino.entity.store.relational.jdbcPassword` | The password that the `JDBCBackend` needs to use when connecting the database. It is required for `MySQL`. | (none) | Yes if you use `JdbcBackend` | 0.5.0 | :::caution We strongly recommend that you change the default value of `gravitino.entity.store.kv.rocksdbPath`, as it's under the deployment directory and future version upgrades may remove it. From ea83dccf36d5eebce2e65005d1633e44557ec5e0 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Fri, 2 Feb 2024 15:11:36 +0800 Subject: [PATCH 21/34] add the code skeleton for mysql backend operating metalake --- core/build.gradle.kts | 2 + .../mysql/mapper/MetalakeMetaMapper.java | 85 +++++++++++ .../mysql/orm/SqlSessionFactoryHelper.java | 72 +++++++++ .../relation/mysql/orm/SqlSessions.java | 65 ++++++++ .../storage/relation/mysql/po/MetalakePO.java | 141 ++++++++++++++++++ .../relation/mysql/utils/POConverters.java | 42 ++++++ .../mysql_init/mysql_backend_init.sql | 13 ++ 7 files changed, 420 insertions(+) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java create mode 100644 core/src/main/resources/mysql_init/mysql_backend_init.sql diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 6f6fb3b7266..a327db37eb6 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { implementation(libs.caffeine) implementation(libs.commons.dbcp2) implementation(libs.commons.io) + implementation(libs.commons.dbcp2) implementation(libs.commons.lang3) implementation(libs.guava) implementation(libs.mybatis) @@ -26,6 +27,7 @@ dependencies { .because("Brings in Guava for Android, which we don't want (and breaks multimaps).") } implementation(libs.rocksdbjni) + implementation(libs.mybatis) annotationProcessor(libs.lombok) compileOnly(libs.lombok) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java new file mode 100644 index 00000000000..2ab836d6578 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java @@ -0,0 +1,85 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.mapper; + +import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import java.util.List; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +public interface MetalakeMetaMapper { + String TABLE_NAME = "metalake_meta"; + + @Select( + "SELECT id, metalake_name as metalakeName, metalake_comment as metalakeComment," + + " properties, audit_info as auditInfo, schema_version as schemaVersion" + + " FROM " + + TABLE_NAME) + List listMetalakePOs(); + + @Select( + "SELECT id, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion" + + " FROM " + + TABLE_NAME + + " WHERE metalake_name = #{metalakeName}") + MetalakePO selectMetalakeMetaByName(@Param("metalakeName") String name); + + @Select("SELECT id FROM " + TABLE_NAME + " WHERE metalake_name = #{metalakeName}") + Long selectMetalakeIdMetaByName(@Param("metalakeName") String name); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" + + " VALUES(" + + " #{metalakeMeta.id}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}" + + " )") + void insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" + + " VALUES(" + + " #{metalakeMeta.id}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}") + void insertMetalakeMetaWithUpdate(@Param("metalakeMeta") MetalakePO metalakePO); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}" + + " WHERE id = #{metalakeMeta.id}") + void updateMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); + + @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") + Integer deleteMetalakeMetaById(@Param("id") Long id); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java new file mode 100644 index 00000000000..9985bc5ab4d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java @@ -0,0 +1,72 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.orm; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; +import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; +import com.google.common.base.Preconditions; +import java.time.Duration; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.pool2.impl.BaseObjectPoolConfig; +import org.apache.ibatis.mapping.Environment; +import org.apache.ibatis.session.Configuration; +import org.apache.ibatis.session.SqlSessionFactory; +import org.apache.ibatis.session.SqlSessionFactoryBuilder; +import org.apache.ibatis.transaction.TransactionFactory; +import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; + +public class SqlSessionFactoryHelper { + private static volatile SqlSessionFactory sqlSessionFactory; + private static final SqlSessionFactoryHelper INSTANCE = new SqlSessionFactoryHelper(); + + public static SqlSessionFactoryHelper getInstance() { + return INSTANCE; + } + + @SuppressWarnings("deprecation") + public void init(Config config) { + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setUrl(config.getRawString(Configs.MYSQL_ENTITY_STORE_URL_KEY)); + dataSource.setDriverClassName(config.getRawString(Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)); + dataSource.setUsername(config.getRawString(Configs.MYSQL_ENTITY_STORE_USERNAME_KEY, "")); + dataSource.setPassword(config.getRawString(Configs.MYSQL_ENTITY_STORE_PASSWORD_KEY, "")); + // close the auto commit, so that need manual commit + dataSource.setDefaultAutoCommit(false); + dataSource.setMaxWaitMillis(1000L); + dataSource.setMaxTotal(20); + dataSource.setMaxIdle(5); + dataSource.setMinIdle(0); + dataSource.setLogAbandoned(true); + dataSource.setRemoveAbandonedOnBorrow(true); + dataSource.setRemoveAbandonedTimeout(60); + dataSource.setTimeBetweenEvictionRunsMillis(Duration.ofMillis(10 * 60 * 1000L).toMillis()); + dataSource.setTestOnBorrow(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW); + dataSource.setTestWhileIdle(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE); + dataSource.setMinEvictableIdleTimeMillis(1000); + dataSource.setNumTestsPerEvictionRun(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN); + dataSource.setTestOnReturn(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN); + dataSource.setSoftMinEvictableIdleTimeMillis( + BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME.toMillis()); + dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO); + TransactionFactory transactionFactory = new JdbcTransactionFactory(); + Environment environment = new Environment("development", transactionFactory, dataSource); + Configuration configuration = new Configuration(environment); + configuration.addMapper(MetalakeMetaMapper.class); + if (sqlSessionFactory == null) { + synchronized (SqlSessionFactoryHelper.class) { + if (sqlSessionFactory == null) { + sqlSessionFactory = new SqlSessionFactoryBuilder().build(configuration); + } + } + } + } + + public SqlSessionFactory getSqlSessionFactory() { + Preconditions.checkState(sqlSessionFactory != null, "SqlSessionFactory is not initialized."); + return sqlSessionFactory; + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java new file mode 100644 index 00000000000..13db581d41a --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java @@ -0,0 +1,65 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.orm; + +import java.io.Closeable; +import java.io.IOException; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.TransactionIsolationLevel; + +public final class SqlSessions implements Closeable { + private static final ThreadLocal sessions = new ThreadLocal<>(); + + private SqlSessions() {} + + public static SqlSession getSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession == null) { + sqlSession = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .openSession(TransactionIsolationLevel.READ_COMMITTED); + sessions.set(sqlSession); + return sqlSession; + } + return sqlSession; + } + + public static void commitAndCloseSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession != null) { + sqlSession.commit(); + sqlSession.close(); + sessions.remove(); + } + } + + public static void rollbackAndCloseSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession != null) { + sqlSession.rollback(); + sqlSession.close(); + sessions.remove(); + } + } + + public static void closeSqlSession() { + SqlSession sqlSession = sessions.get(); + if (sqlSession != null) { + sqlSession.close(); + sessions.remove(); + } + } + + public static T getMapper(Class className) { + return (T) getSqlSession().getMapper(className); + } + + @Override + public void close() throws IOException { + sessions.remove(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java new file mode 100644 index 00000000000..11328b8083d --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java @@ -0,0 +1,141 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.po; + +import com.datastrato.gravitino.json.JsonUtils; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Objects; +import java.util.Map; + +public class MetalakePO { + private Long id; + private String metalakeName; + private String metalakeComment; + private String properties; + private String auditInfo; + private String schemaVersion; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getMetalakeName() { + return metalakeName; + } + + public void setMetalakeName(String metalakeName) { + this.metalakeName = metalakeName; + } + + public String getMetalakeComment() { + return metalakeComment; + } + + public void setMetalakeComment(String metalakeComment) { + this.metalakeComment = metalakeComment; + } + + public String getProperties() { + return properties; + } + + public void setProperties(String properties) { + this.properties = properties; + } + + public String getAuditInfo() { + return auditInfo; + } + + public void setAuditInfo(String auditInfo) { + this.auditInfo = auditInfo; + } + + public String getSchemaVersion() { + return schemaVersion; + } + + public void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MetalakePO)) { + return false; + } + MetalakePO that = (MetalakePO) o; + return Objects.equal(getId(), that.getId()) + && Objects.equal(getMetalakeName(), that.getMetalakeName()) + && Objects.equal(getMetalakeComment(), that.getMetalakeComment()) + && Objects.equal(getProperties(), that.getProperties()) + && Objects.equal(getAuditInfo(), that.getAuditInfo()) + && Objects.equal(getSchemaVersion(), that.getSchemaVersion()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getId(), + getMetalakeName(), + getMetalakeComment(), + getProperties(), + getAuditInfo(), + getSchemaVersion()); + } + + public static class Builder { + private final MetalakePO metalakePO; + + public Builder() { + metalakePO = new MetalakePO(); + } + + public MetalakePO.Builder withId(Long id) { + metalakePO.id = id; + return this; + } + + public MetalakePO.Builder withMetalakeName(String name) { + metalakePO.metalakeName = name; + return this; + } + + public MetalakePO.Builder withMetalakeComment(String comment) { + metalakePO.metalakeComment = comment; + return this; + } + + public MetalakePO.Builder withProperties(Map properties) + throws JsonProcessingException { + metalakePO.properties = JsonUtils.objectMapper().writeValueAsString(properties); + return this; + } + + public MetalakePO.Builder withAuditInfo(AuditInfo auditInfo) throws JsonProcessingException { + metalakePO.auditInfo = JsonUtils.objectMapper().writeValueAsString(auditInfo); + return this; + } + + public MetalakePO.Builder withVersion(SchemaVersion version) throws JsonProcessingException { + metalakePO.schemaVersion = JsonUtils.objectMapper().writeValueAsString(version); + return this; + } + + public MetalakePO build() { + return metalakePO; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java new file mode 100644 index 00000000000..0c366b847fb --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java @@ -0,0 +1,42 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation.mysql.utils; + +import com.datastrato.gravitino.json.JsonUtils; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.Map; + +public class POConverters { + private POConverters() {} + + public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) throws JsonProcessingException { + return new MetalakePO.Builder() + .withId(baseMetalake.id()) + .withMetalakeName(baseMetalake.name()) + .withMetalakeComment(baseMetalake.comment()) + .withProperties(baseMetalake.properties()) + .withAuditInfo((AuditInfo) baseMetalake.auditInfo()) + .withVersion(baseMetalake.getVersion()) + .build(); + } + + public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) throws JsonProcessingException { + return new BaseMetalake.Builder() + .withId(metalakePO.getId()) + .withName(metalakePO.getMetalakeName()) + .withComment(metalakePO.getMetalakeComment()) + .withProperties(JsonUtils.objectMapper().readValue(metalakePO.getProperties(), Map.class)) + .withAuditInfo( + JsonUtils.objectMapper().readValue(metalakePO.getAuditInfo(), AuditInfo.class)) + .withVersion( + JsonUtils.objectMapper().readValue(metalakePO.getSchemaVersion(), SchemaVersion.class)) + .build(); + } +} diff --git a/core/src/main/resources/mysql_init/mysql_backend_init.sql b/core/src/main/resources/mysql_init/mysql_backend_init.sql new file mode 100644 index 00000000000..1e9c0760410 --- /dev/null +++ b/core/src/main/resources/mysql_init/mysql_backend_init.sql @@ -0,0 +1,13 @@ +CREATE DATABASE IF NOT EXISTS `gravitino_meta` DEFAULT CHARACTER SET utf8mb4; +USE `gravitino_meta`; +CREATE TABLE IF NOT EXISTS `metalake_meta` +( + `id` bigint(20) unsigned NOT NULL COMMENT 'metalake id', + `metalake_name` varchar(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` varchar(256) DEFAULT '' COMMENT 'metalake comment', + `properties` mediumtext DEFAULT NULL COMMENT 'metalake properties', + `audit_info` mediumtext NOT NULL COMMENT 'metalake audit info', + `schema_version` text NOT NULL COMMENT 'metalake schema version info', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_mn` (`metalake_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'metalake metadata'; \ No newline at end of file From 97a00c7fec3386b2513cd88fab5c9a0100906128 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 6 Feb 2024 15:16:30 +0800 Subject: [PATCH 22/34] fix comments and uts --- .../mysql/utils/TestPOConverters.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java new file mode 100644 index 00000000000..3612f099466 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java @@ -0,0 +1,93 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.datastrato.gravitino.json.JsonUtils; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class TestPOConverters { + private static final LocalDateTime FIX_DATE_TIME = LocalDateTime.of(2024, 2, 6, 0, 0, 0); + + private static final Instant FIX_INSTANT = FIX_DATE_TIME.toInstant(ZoneOffset.UTC); + + @Test + public void testFromMetalakePO() throws JsonProcessingException { + MetalakePO metalakePO = createMetalakePO(1L, "test", "this is test"); + + BaseMetalake expectedMetalake = createMetalake(1L, "test", "this is test"); + + BaseMetalake convertedMetalake = POConverters.fromMetalakePO(metalakePO); + + // Assert + assertEquals(expectedMetalake.id(), convertedMetalake.id()); + assertEquals(expectedMetalake.name(), convertedMetalake.name()); + assertEquals(expectedMetalake.comment(), convertedMetalake.comment()); + assertEquals( + expectedMetalake.properties().get("key"), convertedMetalake.properties().get("key")); + assertEquals(expectedMetalake.auditInfo().creator(), convertedMetalake.auditInfo().creator()); + assertEquals(expectedMetalake.getVersion(), convertedMetalake.getVersion()); + } + + @Test + public void testToMetalakePO() throws JsonProcessingException { + BaseMetalake metalake = createMetalake(1L, "test", "this is test"); + + MetalakePO expectedMetalakePO = createMetalakePO(1L, "test", "this is test"); + + MetalakePO actualMetalakePO = POConverters.toMetalakePO(metalake); + + // Assert + assertEquals(expectedMetalakePO.getId(), actualMetalakePO.getId()); + assertEquals(expectedMetalakePO.getMetalakeName(), actualMetalakePO.getMetalakeName()); + assertEquals(expectedMetalakePO.getMetalakeComment(), actualMetalakePO.getMetalakeComment()); + assertEquals(expectedMetalakePO.getProperties(), actualMetalakePO.getProperties()); + assertEquals(expectedMetalakePO.getAuditInfo(), actualMetalakePO.getAuditInfo()); + assertEquals(expectedMetalakePO.getSchemaVersion(), actualMetalakePO.getSchemaVersion()); + } + + private static BaseMetalake createMetalake(Long id, String name, String comment) { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); + Map properties = new HashMap<>(); + properties.put("key", "value"); + return new BaseMetalake.Builder() + .withId(id) + .withName(name) + .withComment(comment) + .withProperties(properties) + .withAuditInfo(auditInfo) + .withVersion(SchemaVersion.V_0_1) + .build(); + } + + private static MetalakePO createMetalakePO(Long id, String name, String comment) + throws JsonProcessingException { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); + Map properties = new HashMap<>(); + properties.put("key", "value"); + return new MetalakePO.Builder() + .withId(id) + .withMetalakeName(name) + .withMetalakeComment(comment) + .withProperties(JsonUtils.objectMapper().writeValueAsString(properties)) + .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(auditInfo)) + .withVersion(JsonUtils.objectMapper().writeValueAsString(SchemaVersion.V_0_1)) + .build(); + } +} From 5ad6b8d2374a124bb02bae41242f6ba4de9e0606 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 6 Feb 2024 19:18:06 +0800 Subject: [PATCH 23/34] fix some comments --- .../relational/mysql/utils/SessionUtils.java | 78 +++++++++++++++++++ .../mysql/utils/TestPOConverters.java | 6 +- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java new file mode 100644 index 00000000000..2b6869f9440 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java @@ -0,0 +1,78 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.utils; + +import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessions; +import java.util.Arrays; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.ibatis.session.SqlSession; + +public class SessionUtils { + private SessionUtils() {} + + public static void doWithCommit(Class mapperClazz, Consumer consumer) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + consumer.accept(mapper); + SqlSessions.commitAndCloseSqlSession(); + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } + } + + public static R doWithCommitAndFetchResult(Class mapperClazz, Function func) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + R result = func.apply(mapper); + SqlSessions.commitAndCloseSqlSession(); + return result; + } catch (Throwable t) { + throw new RuntimeException(t); + } finally { + SqlSessions.closeSqlSession(); + } + } + } + + public static void doWithoutCommit(Class mapperClazz, Consumer consumer) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + consumer.accept(mapper); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + + public static R getWithoutCommit(Class mapperClazz, Function func) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + T mapper = SqlSessions.getMapper(mapperClazz); + return func.apply(mapper); + } catch (Throwable t) { + throw new RuntimeException(t); + } finally { + SqlSessions.closeSqlSession(); + } + } + } + + public static void doMultipleWithCommit(Runnable... operations) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + Arrays.stream(operations).forEach(Runnable::run); + SqlSessions.commitAndCloseSqlSession(); + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java index 3612f099466..27f49756626 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java @@ -85,9 +85,9 @@ private static MetalakePO createMetalakePO(Long id, String name, String comment) .withId(id) .withMetalakeName(name) .withMetalakeComment(comment) - .withProperties(JsonUtils.objectMapper().writeValueAsString(properties)) - .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(auditInfo)) - .withVersion(JsonUtils.objectMapper().writeValueAsString(SchemaVersion.V_0_1)) + .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(properties)) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(auditInfo)) + .withVersion(JsonUtils.anyFieldMapper().writeValueAsString(SchemaVersion.V_0_1)) .build(); } } From 723541f90619c1558a406859e10a3e99ee206e9c Mon Sep 17 00:00:00 2001 From: Jiebao Xiao Date: Sat, 17 Feb 2024 14:43:52 +0800 Subject: [PATCH 24/34] rename to jdbc backend --- .../relational/mysql/utils/SessionUtils.java | 78 ---------------- .../mysql/utils/TestPOConverters.java | 93 ------------------- 2 files changed, 171 deletions(-) delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java delete mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java deleted file mode 100644 index 2b6869f9440..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mysql/utils/SessionUtils.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relational.mysql.utils; - -import com.datastrato.gravitino.storage.relational.mysql.session.SqlSessions; -import java.util.Arrays; -import java.util.function.Consumer; -import java.util.function.Function; -import org.apache.ibatis.session.SqlSession; - -public class SessionUtils { - private SessionUtils() {} - - public static void doWithCommit(Class mapperClazz, Consumer consumer) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - T mapper = SqlSessions.getMapper(mapperClazz); - consumer.accept(mapper); - SqlSessions.commitAndCloseSqlSession(); - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); - } - } - } - - public static R doWithCommitAndFetchResult(Class mapperClazz, Function func) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - T mapper = SqlSessions.getMapper(mapperClazz); - R result = func.apply(mapper); - SqlSessions.commitAndCloseSqlSession(); - return result; - } catch (Throwable t) { - throw new RuntimeException(t); - } finally { - SqlSessions.closeSqlSession(); - } - } - } - - public static void doWithoutCommit(Class mapperClazz, Consumer consumer) { - try { - T mapper = SqlSessions.getMapper(mapperClazz); - consumer.accept(mapper); - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - - public static R getWithoutCommit(Class mapperClazz, Function func) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - T mapper = SqlSessions.getMapper(mapperClazz); - return func.apply(mapper); - } catch (Throwable t) { - throw new RuntimeException(t); - } finally { - SqlSessions.closeSqlSession(); - } - } - } - - public static void doMultipleWithCommit(Runnable... operations) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - Arrays.stream(operations).forEach(Runnable::run); - SqlSessions.commitAndCloseSqlSession(); - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); - } - } - } -} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java deleted file mode 100644 index 27f49756626..00000000000 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/utils/TestPOConverters.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relational.mysql.utils; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.datastrato.gravitino.json.JsonUtils; -import com.datastrato.gravitino.meta.AuditInfo; -import com.datastrato.gravitino.meta.BaseMetalake; -import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relational.mysql.po.MetalakePO; -import com.fasterxml.jackson.core.JsonProcessingException; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.HashMap; -import java.util.Map; -import org.junit.jupiter.api.Test; - -public class TestPOConverters { - private static final LocalDateTime FIX_DATE_TIME = LocalDateTime.of(2024, 2, 6, 0, 0, 0); - - private static final Instant FIX_INSTANT = FIX_DATE_TIME.toInstant(ZoneOffset.UTC); - - @Test - public void testFromMetalakePO() throws JsonProcessingException { - MetalakePO metalakePO = createMetalakePO(1L, "test", "this is test"); - - BaseMetalake expectedMetalake = createMetalake(1L, "test", "this is test"); - - BaseMetalake convertedMetalake = POConverters.fromMetalakePO(metalakePO); - - // Assert - assertEquals(expectedMetalake.id(), convertedMetalake.id()); - assertEquals(expectedMetalake.name(), convertedMetalake.name()); - assertEquals(expectedMetalake.comment(), convertedMetalake.comment()); - assertEquals( - expectedMetalake.properties().get("key"), convertedMetalake.properties().get("key")); - assertEquals(expectedMetalake.auditInfo().creator(), convertedMetalake.auditInfo().creator()); - assertEquals(expectedMetalake.getVersion(), convertedMetalake.getVersion()); - } - - @Test - public void testToMetalakePO() throws JsonProcessingException { - BaseMetalake metalake = createMetalake(1L, "test", "this is test"); - - MetalakePO expectedMetalakePO = createMetalakePO(1L, "test", "this is test"); - - MetalakePO actualMetalakePO = POConverters.toMetalakePO(metalake); - - // Assert - assertEquals(expectedMetalakePO.getId(), actualMetalakePO.getId()); - assertEquals(expectedMetalakePO.getMetalakeName(), actualMetalakePO.getMetalakeName()); - assertEquals(expectedMetalakePO.getMetalakeComment(), actualMetalakePO.getMetalakeComment()); - assertEquals(expectedMetalakePO.getProperties(), actualMetalakePO.getProperties()); - assertEquals(expectedMetalakePO.getAuditInfo(), actualMetalakePO.getAuditInfo()); - assertEquals(expectedMetalakePO.getSchemaVersion(), actualMetalakePO.getSchemaVersion()); - } - - private static BaseMetalake createMetalake(Long id, String name, String comment) { - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); - Map properties = new HashMap<>(); - properties.put("key", "value"); - return new BaseMetalake.Builder() - .withId(id) - .withName(name) - .withComment(comment) - .withProperties(properties) - .withAuditInfo(auditInfo) - .withVersion(SchemaVersion.V_0_1) - .build(); - } - - private static MetalakePO createMetalakePO(Long id, String name, String comment) - throws JsonProcessingException { - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); - Map properties = new HashMap<>(); - properties.put("key", "value"); - return new MetalakePO.Builder() - .withId(id) - .withMetalakeName(name) - .withMetalakeComment(comment) - .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(properties)) - .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(auditInfo)) - .withVersion(JsonUtils.anyFieldMapper().writeValueAsString(SchemaVersion.V_0_1)) - .build(); - } -} From f004dc30108ad8c1cfbfbdd4ab7e9aef440f88f7 Mon Sep 17 00:00:00 2001 From: Jiebao Xiao Date: Sat, 17 Feb 2024 17:13:36 +0800 Subject: [PATCH 25/34] fix comments --- .../com/datastrato/gravitino/storage/relational/JDBCBackend.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index d7825445bcd..1695af21779 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -6,7 +6,6 @@ package com.datastrato.gravitino.storage.relational; import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; From 3057e0b82f43140f95203fc6be48da252090cf7e Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 6 Feb 2024 14:16:29 +0800 Subject: [PATCH 26/34] fix comments --- LICENSE.bin | 1 + .../session/TestSqlSessionFactoryHelper.java | 102 ++++++++++++++++++ .../mysql/session/TestSqlSessions.java | 83 ++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java diff --git a/LICENSE.bin b/LICENSE.bin index 3456cec1422..31de2f1c616 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -411,6 +411,7 @@ Mozilla Public License Javassist + H2 Database Engine This product bundles various third-party components also placed in diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java new file mode 100644 index 00000000000..1ebeac5c2f0 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.session; + +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.datastrato.gravitino.Config; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.UUID; +import org.apache.commons.dbcp2.BasicDataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestSqlSessionFactoryHelper { + private static final String MYSQL_STORE_PATH = + "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; + + private static Config config; + + @BeforeAll + public static void setUp() { + File dir = new File(DB_DIR); + if (dir.exists() || !dir.isDirectory()) { + dir.delete(); + } + dir.mkdirs(); + + config = Mockito.mock(Config.class); + Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + .thenReturn("org.h2.Driver"); + } + + @BeforeEach + public void init() { + SqlSessionFactoryHelper.setSqlSessionFactory(null); + } + + @AfterEach + public void cleanUp() { + SqlSessionFactoryHelper.setSqlSessionFactory(null); + } + + @AfterAll + public static void tearDown() throws IOException { + File dir = new File(DB_DIR); + if (dir.exists()) { + dir.delete(); + } + } + + @Test + public void testGetInstance() { + SqlSessionFactoryHelper instance = SqlSessionFactoryHelper.getInstance(); + assertNotNull(instance); + } + + @Test + public void testInit() throws SQLException { + SqlSessionFactoryHelper.getInstance().init(config); + assertNotNull(SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); + BasicDataSource dataSource = + (BasicDataSource) + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getEnvironment() + .getDataSource(); + assertEquals("org.h2.Driver", dataSource.getDriverClassName()); + assertEquals(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY), dataSource.getUrl()); + } + + @Test + public void testGetSqlSessionFactoryWithoutInit() { + assertThrows( + IllegalStateException.class, + () -> SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java new file mode 100644 index 00000000000..57c4a39e606 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java @@ -0,0 +1,83 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mysql.session; + +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.datastrato.gravitino.Config; +import java.io.File; +import java.io.IOException; +import java.util.UUID; +import org.apache.ibatis.session.SqlSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestSqlSessions { + private static final String MYSQL_STORE_PATH = + "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; + + @BeforeAll + public static void setUp() { + File dir = new File(DB_DIR); + if (dir.exists() || !dir.isDirectory()) { + dir.delete(); + } + dir.mkdirs(); + + Config config = Mockito.mock(Config.class); + Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + .thenReturn("org.h2.Driver"); + SqlSessionFactoryHelper.getInstance().init(config); + } + + @AfterAll + public static void tearDown() throws IOException { + File dir = new File(DB_DIR); + if (dir.exists()) { + dir.delete(); + } + } + + @Test + public void testOpenAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.closeSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } + + @Test + public void testOpenAndCommitAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.commitAndCloseSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } + + @Test + public void testOpenAndRollbackAndCloseSqlSession() { + SqlSession session = SqlSessions.getSqlSession(); + assertNotNull(session); + SqlSessions.rollbackAndCloseSqlSession(); + assertNull(SqlSessions.getSessions().get()); + } +} From 51f45f440e8efd51ae8ad95a558fa793d2815479 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Fri, 2 Feb 2024 15:11:36 +0800 Subject: [PATCH 27/34] add the code skeleton for mysql backend operating metalake --- .../com/datastrato/gravitino/Configs.java | 5 + .../storage/relation/mysql/MySQLBackend.java | 222 ++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 384f47628bf..f92ced76698 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -28,6 +28,11 @@ public interface Configs { String ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY = "gravitino.entity.store.relational.jdbcPassword"; + String MYSQL_ENTITY_STORE_URL_KEY = "gravitino.entity.store.mysql.url"; + String MYSQL_ENTITY_STORE_DRIVER_NAME_KEY = "gravitino.entity.store.mysql.driverName"; + String MYSQL_ENTITY_STORE_USERNAME_KEY = "gravitino.entity.store.mysql.username"; + String MYSQL_ENTITY_STORE_PASSWORD_KEY = "gravitino.entity.store.mysql.password"; + String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; Long DEFAULT_KV_DELETE_AFTER_TIME = 604800000L; // 7 days diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java new file mode 100644 index 00000000000..1ff8dca7f39 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java @@ -0,0 +1,222 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relation.mysql; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.AlreadyExistsException; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.storage.relation.RelationBackend; +import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessionFactoryHelper; +import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessions; +import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; +import com.datastrato.gravitino.storage.relation.mysql.utils.POConverters; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.ibatis.session.SqlSession; + +/** + * {@link MySQLBackend} is a MySQL implementation of RelationBackend interface. If we want to use + * another relation implementation, We can just implement {@link RelationBackend} interface and use + * it in the Gravitino. + */ +public class MySQLBackend implements RelationBackend { + + /** Initialize the MySQL backend instance. */ + @Override + public void initialize(Config config) { + SqlSessionFactoryHelper.getInstance().init(config); + } + + @Override + public List list( + Namespace namespace, Entity.EntityType entityType) { + try (SqlSession session = SqlSessions.getSqlSession()) { + switch (entityType) { + case METALAKE: + List metalakePOS = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .listMetalakePOs(); + return (List) + metalakePOS.stream() + .map( + metalakePO -> { + try { + return POConverters.fromMetalakePO(metalakePO); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for list operation", entityType)); + } + } finally { + SqlSessions.closeSqlSession(); + } + } + + @Override + public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { + try (SqlSession session = SqlSessions.getSqlSession()) { + switch (entityType) { + case METALAKE: + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + return metalakePO != null; + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for exists operation", entityType)); + } + } finally { + SqlSessions.closeSqlSession(); + } + } + + @Override + public void insert(E e, boolean overwritten) + throws EntityAlreadyExistsException { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + if (e instanceof BaseMetalake) { + MetalakePO metalakePO = POConverters.toMetalakePO((BaseMetalake) e); + if (overwritten) { + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .insertMetalakeMetaWithUpdate(metalakePO); + } else { + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .insertMetalakeMeta(metalakePO); + } + SqlSessions.commitAndCloseSqlSession(); + } else { + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for put operation", e.getClass())); + } + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } + } + + @Override + public E update( + NameIdentifier ident, Entity.EntityType entityType, Function updater) + throws NoSuchEntityException, AlreadyExistsException { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + switch (entityType) { + case METALAKE: + BaseMetalake oldMetalakeEntity = + POConverters.fromMetalakePO( + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name())); + BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); + Preconditions.checkArgument( + Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), + String.format( + "The updated metalake entity id: %s is not same with the metalake entity id before: %s", + newMetalakeEntity.id(), oldMetalakeEntity.id())); + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity)); + SqlSessions.commitAndCloseSqlSession(); + return (E) newMetalakeEntity; + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for update operation", entityType)); + } + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } + } + + @Override + public E get( + NameIdentifier ident, Entity.EntityType entityType) + throws NoSuchEntityException, IOException { + try (SqlSession session = SqlSessions.getSqlSession()) { + switch (entityType) { + case METALAKE: + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + return (E) POConverters.fromMetalakePO(metalakePO); + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for get operation", entityType)); + } + } finally { + SqlSessions.closeSqlSession(); + } + } + + @Override + public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { + try (SqlSession session = SqlSessions.getSqlSession()) { + try { + switch (entityType) { + case METALAKE: + Long metalakeId = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeIdMetaByName(ident.name()); + if (metalakeId != null) { + // delete metalake + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .deleteMetalakeMetaById(metalakeId); + if (cascade) { + // TODO We will cascade delete the metadata of sub-resources under metalake + } + SqlSessions.commitAndCloseSqlSession(); + } + return true; + case CATALOG: + case SCHEMA: + case TABLE: + case FILESET: + default: + throw new IllegalArgumentException( + String.format("Unsupported entity type: %s for delete operation", entityType)); + } + } catch (Throwable t) { + SqlSessions.rollbackAndCloseSqlSession(); + throw new RuntimeException(t); + } + } + } + + @Override + public void close() throws IOException {} +} From 38403be26cb2272f87a14ae07f2a3b8083620361 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Sun, 4 Feb 2024 11:09:45 +0800 Subject: [PATCH 28/34] fix update --- .../storage/relation/mysql/MySQLBackend.java | 10 ++++----- .../mysql/mapper/MetalakeMetaMapper.java | 21 ++++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java index 1ff8dca7f39..7f121d3b7dc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java @@ -130,10 +130,10 @@ public E update( try { switch (entityType) { case METALAKE: - BaseMetalake oldMetalakeEntity = - POConverters.fromMetalakePO( - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name())); + MetalakePO oldMetalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(ident.name()); + BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); Preconditions.checkArgument( Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), @@ -141,7 +141,7 @@ public E update( "The updated metalake entity id: %s is not same with the metalake entity id before: %s", newMetalakeEntity.id(), oldMetalakeEntity.id())); ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity)); + .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO); SqlSessions.commitAndCloseSqlSession(); return (E) newMetalakeEntity; case CATALOG: diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java index 2ab836d6578..5a34889481c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java @@ -72,13 +72,20 @@ public interface MetalakeMetaMapper { @Update( "UPDATE " + TABLE_NAME - + " SET metalake_name = #{metalakeMeta.metalakeName}," - + " metalake_comment = #{metalakeMeta.metalakeComment}," - + " properties = #{metalakeMeta.properties}," - + " audit_info = #{metalakeMeta.auditInfo}," - + " schema_version = #{metalakeMeta.schemaVersion}" - + " WHERE id = #{metalakeMeta.id}") - void updateMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); + + " SET metalake_name = #{newMetalakeMeta.metalakeName}," + + " metalake_comment = #{newMetalakeMeta.metalakeComment}," + + " properties = #{newMetalakeMeta.properties}," + + " audit_info = #{newMetalakeMeta.auditInfo}," + + " schema_version = #{newMetalakeMeta.schemaVersion}" + + " WHERE id = #{oldMetalakeMeta.id}" + + " and metalake_name = #{oldMetalakeMeta.metalakeComment}" + + " and metalake_comment = #{oldMetalakeMeta.metalakeComment}" + + " and properties = #{oldMetalakeMeta.properties}" + + " and audit_info = #{oldMetalakeMeta.auditInfo}" + + " and schema_version = #{oldMetalakeMeta.schemaVersion}") + void updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") Integer deleteMetalakeMetaById(@Param("id") Long id); From 074020008735777dcf34f28bdcce92f41fb6090f Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 5 Feb 2024 11:55:32 +0800 Subject: [PATCH 29/34] add unit test --- .../storage/relation/mysql/MySQLBackend.java | 31 +- .../mysql/mapper/MetalakeMetaMapper.java | 2 +- .../mysql/orm/SqlSessionFactoryHelper.java | 12 + .../relation/mysql/orm/SqlSessions.java | 36 ++- .../storage/relation/mysql/po/MetalakePO.java | 17 +- .../relation/mysql/utils/POConverters.java | 21 +- .../mysql_init/mysql_backend_init.sql | 13 - .../relation/TestRelationEntityStore.java | 290 ++++++++++++++++++ 8 files changed, 377 insertions(+), 45 deletions(-) delete mode 100644 core/src/main/resources/mysql_init/mysql_backend_init.sql create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java index 7f121d3b7dc..097b9c3ff82 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -50,17 +51,18 @@ public List list( List metalakePOS = ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) .listMetalakePOs(); - return (List) - metalakePOS.stream() + return metalakePOS != null + ? metalakePOS.stream() .map( metalakePO -> { try { - return POConverters.fromMetalakePO(metalakePO); + return (E) POConverters.fromMetalakePO(metalakePO); } catch (JsonProcessingException e) { throw new RuntimeException(e); } }) - .collect(Collectors.toList()); + .collect(Collectors.toList()) + : new ArrayList<>(); case CATALOG: case SCHEMA: case TABLE: @@ -102,16 +104,24 @@ public void insert(E e, boolean overwritten) try (SqlSession session = SqlSessions.getSqlSession()) { try { if (e instanceof BaseMetalake) { - MetalakePO metalakePO = POConverters.toMetalakePO((BaseMetalake) e); + MetalakePO metalakePO = + ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) + .selectMetalakeMetaByName(e.nameIdentifier().name()); + if (!overwritten && metalakePO != null) { + throw new EntityAlreadyExistsException( + String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); + } + if (overwritten) { ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMetaWithUpdate(metalakePO); + .insertMetalakeMetaWithUpdate(POConverters.toMetalakePO((BaseMetalake) e)); } else { ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMeta(metalakePO); + .insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); } SqlSessions.commitAndCloseSqlSession(); } else { + SqlSessions.closeSqlSession(); throw new IllegalArgumentException( String.format("Unsupported entity type: %s for put operation", e.getClass())); } @@ -138,7 +148,7 @@ public E update( Preconditions.checkArgument( Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), String.format( - "The updated metalake entity id: %s is not same with the metalake entity id before: %s", + "The updated metalake entity id: %s should same with the metalake entity id before: %s", newMetalakeEntity.id(), oldMetalakeEntity.id())); ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO); @@ -169,6 +179,9 @@ public E get( MetalakePO metalakePO = ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) .selectMetalakeMetaByName(ident.name()); + if (metalakePO == null) { + return null; + } return (E) POConverters.fromMetalakePO(metalakePO); case CATALOG: case SCHEMA: @@ -197,7 +210,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) .deleteMetalakeMetaById(metalakeId); if (cascade) { - // TODO We will cascade delete the metadata of sub-resources under metalake + // TODO We will cascade delete the metadata of sub-resources under the metalake } SqlSessions.commitAndCloseSqlSession(); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java index 5a34889481c..5f863f88328 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java @@ -78,7 +78,7 @@ public interface MetalakeMetaMapper { + " audit_info = #{newMetalakeMeta.auditInfo}," + " schema_version = #{newMetalakeMeta.schemaVersion}" + " WHERE id = #{oldMetalakeMeta.id}" - + " and metalake_name = #{oldMetalakeMeta.metalakeComment}" + + " and metalake_name = #{oldMetalakeMeta.metalakeName}" + " and metalake_comment = #{oldMetalakeMeta.metalakeComment}" + " and properties = #{oldMetalakeMeta.properties}" + " and audit_info = #{oldMetalakeMeta.auditInfo}" diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java index 9985bc5ab4d..67162dd761c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java @@ -19,6 +19,11 @@ import org.apache.ibatis.transaction.TransactionFactory; import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; +/** + * SqlSessionFactoryHelper maintains the MyBatis's {@link SqlSessionFactory} object, which is used + * to create the {@link org.apache.ibatis.session.SqlSession} object. It is a singleton class and + * should be initialized only once. + */ public class SqlSessionFactoryHelper { private static volatile SqlSessionFactory sqlSessionFactory; private static final SqlSessionFactoryHelper INSTANCE = new SqlSessionFactoryHelper(); @@ -27,6 +32,13 @@ public static SqlSessionFactoryHelper getInstance() { return INSTANCE; } + private SqlSessionFactoryHelper() {} + + /** + * Initialize the SqlSessionFactory object. + * + * @param config Config object to get the MySQL connection details from the config. + */ @SuppressWarnings("deprecation") public void init(Config config) { BasicDataSource dataSource = new BasicDataSource(); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java index 13db581d41a..5c3398efaa7 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java @@ -5,16 +5,25 @@ package com.datastrato.gravitino.storage.relation.mysql.orm; -import java.io.Closeable; -import java.io.IOException; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.TransactionIsolationLevel; -public final class SqlSessions implements Closeable { +/** + * SqlSessions is a utility class to maintain the MyBatis's {@link SqlSession} object. It is a + * thread local class and should be used to get the {@link SqlSession} object. It also provides the + * methods to commit, rollback and close the {@link SqlSession} object. + */ +public final class SqlSessions { private static final ThreadLocal sessions = new ThreadLocal<>(); private SqlSessions() {} + /** + * Get the SqlSession object. If the SqlSession object is not present in the thread local, then + * create a new SqlSession object and set it in the thread local. + * + * @return SqlSession object from the thread local storage. + */ public static SqlSession getSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession == null) { @@ -28,6 +37,10 @@ public static SqlSession getSqlSession() { return sqlSession; } + /** + * Commit the SqlSession object and close it. It also removes the SqlSession object from the + * thread local storage. + */ public static void commitAndCloseSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { @@ -37,6 +50,10 @@ public static void commitAndCloseSqlSession() { } } + /** + * Rollback the SqlSession object and close it. It also removes the SqlSession object from the + * thread local storage. + */ public static void rollbackAndCloseSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { @@ -46,6 +63,7 @@ public static void rollbackAndCloseSqlSession() { } } + /** Close the SqlSession object and remove it from the thread local storage. */ public static void closeSqlSession() { SqlSession sqlSession = sessions.get(); if (sqlSession != null) { @@ -54,12 +72,14 @@ public static void closeSqlSession() { } } + /** + * Get the Mapper object from the SqlSession object. + * + * @param className the class name of the Mapper object. + * @return the Mapper object. + * @param the type of the Mapper object. + */ public static T getMapper(Class className) { return (T) getSqlSession().getMapper(className); } - - @Override - public void close() throws IOException { - sessions.remove(); - } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java index 11328b8083d..f41e82fbc4a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java @@ -5,12 +5,8 @@ package com.datastrato.gravitino.storage.relation.mysql.po; -import com.datastrato.gravitino.json.JsonUtils; -import com.datastrato.gravitino.meta.AuditInfo; -import com.datastrato.gravitino.meta.SchemaVersion; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Objects; -import java.util.Map; public class MetalakePO { private Long id; @@ -118,19 +114,18 @@ public MetalakePO.Builder withMetalakeComment(String comment) { return this; } - public MetalakePO.Builder withProperties(Map properties) - throws JsonProcessingException { - metalakePO.properties = JsonUtils.objectMapper().writeValueAsString(properties); + public MetalakePO.Builder withProperties(String properties) throws JsonProcessingException { + metalakePO.properties = properties; return this; } - public MetalakePO.Builder withAuditInfo(AuditInfo auditInfo) throws JsonProcessingException { - metalakePO.auditInfo = JsonUtils.objectMapper().writeValueAsString(auditInfo); + public MetalakePO.Builder withAuditInfo(String auditInfo) throws JsonProcessingException { + metalakePO.auditInfo = auditInfo; return this; } - public MetalakePO.Builder withVersion(SchemaVersion version) throws JsonProcessingException { - metalakePO.schemaVersion = JsonUtils.objectMapper().writeValueAsString(version); + public MetalakePO.Builder withVersion(String version) throws JsonProcessingException { + metalakePO.schemaVersion = version; return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java index 0c366b847fb..d15c3a1f918 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java @@ -13,20 +13,35 @@ import com.fasterxml.jackson.core.JsonProcessingException; import java.util.Map; +/** POConverters is a utility class to convert PO to Base and vice versa. */ public class POConverters { private POConverters() {} + /** + * Convert {@link BaseMetalake} to {@link MetalakePO} + * + * @param baseMetalake BaseMetalake object + * @return MetalakePO object from BaseMetalake object + * @throws JsonProcessingException + */ public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) throws JsonProcessingException { return new MetalakePO.Builder() .withId(baseMetalake.id()) .withMetalakeName(baseMetalake.name()) .withMetalakeComment(baseMetalake.comment()) - .withProperties(baseMetalake.properties()) - .withAuditInfo((AuditInfo) baseMetalake.auditInfo()) - .withVersion(baseMetalake.getVersion()) + .withProperties(JsonUtils.objectMapper().writeValueAsString(baseMetalake.properties())) + .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(baseMetalake.auditInfo())) + .withVersion(JsonUtils.objectMapper().writeValueAsString(baseMetalake.getVersion())) .build(); } + /** + * Convert {@link MetalakePO} to {@link BaseMetalake} + * + * @param metalakePO MetalakePO object + * @return BaseMetalake object from MetalakePO object + * @throws JsonProcessingException + */ public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) throws JsonProcessingException { return new BaseMetalake.Builder() .withId(metalakePO.getId()) diff --git a/core/src/main/resources/mysql_init/mysql_backend_init.sql b/core/src/main/resources/mysql_init/mysql_backend_init.sql deleted file mode 100644 index 1e9c0760410..00000000000 --- a/core/src/main/resources/mysql_init/mysql_backend_init.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `gravitino_meta` DEFAULT CHARACTER SET utf8mb4; -USE `gravitino_meta`; -CREATE TABLE IF NOT EXISTS `metalake_meta` -( - `id` bigint(20) unsigned NOT NULL COMMENT 'metalake id', - `metalake_name` varchar(128) NOT NULL COMMENT 'metalake name', - `metalake_comment` varchar(256) DEFAULT '' COMMENT 'metalake comment', - `properties` mediumtext DEFAULT NULL COMMENT 'metalake properties', - `audit_info` mediumtext NOT NULL COMMENT 'metalake audit info', - `schema_version` text NOT NULL COMMENT 'metalake schema version info', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_mn` (`metalake_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'metalake metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java new file mode 100644 index 00000000000..71dd43ac9b9 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java @@ -0,0 +1,290 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relation; + +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATION_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATION_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; +import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; +import static com.datastrato.gravitino.Configs.RELATION_ENTITY_STORE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.EntityStoreFactory; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessionFactoryHelper; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import org.apache.ibatis.session.SqlSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestRelationEntityStore { + private static final String MYSQL_STORE_PATH = + "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; + private static EntityStore entityStore = null; + + @BeforeAll + public static void setUp() { + File dir = new File(DB_DIR); + if (dir.exists() || !dir.isDirectory()) { + dir.delete(); + } + dir.mkdirs(); + + // Use H2 DATABASE to simulate MySQL + Config config = Mockito.mock(Config.class); + Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATION_ENTITY_STORE); + Mockito.when(config.get(ENTITY_RELATION_STORE)).thenReturn(DEFAULT_ENTITY_RELATION_STORE); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); + Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) + .thenReturn("org.h2.Driver"); + entityStore = EntityStoreFactory.createEntityStore(config); + entityStore.initialize(config); + + // Read the ddl sql to create table + String scriptPath = "h2/h2-init.sql"; + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); + Connection connection = sqlSession.getConnection(); + Statement statement = connection.createStatement()) { + URL scriptUrl = ClassLoader.getSystemResource(scriptPath); + if (scriptUrl == null) { + throw new IllegalStateException("Cannot find init sql script:" + scriptPath); + } + StringBuilder ddlBuilder = new StringBuilder(); + try (InputStreamReader inputStreamReader = + new InputStreamReader( + Files.newInputStream(Paths.get(scriptUrl.getPath())), StandardCharsets.UTF_8); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { + String line; + while ((line = bufferedReader.readLine()) != null) { + ddlBuilder.append(line).append("\n"); + } + } + statement.execute(ddlBuilder.toString()); + } catch (Exception e) { + throw new IllegalStateException("Create tables failed", e); + } + } + + @AfterEach + public void destroy() { + truncateAllTables(); + } + + @AfterAll + public static void tearDown() throws IOException { + dropAllTables(); + entityStore.close(); + File dir = new File(DB_DIR); + if (dir.exists()) { + dir.delete(); + } + } + + @Test + public void testPutAndGet() throws IOException { + BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); + entityStore.put(metalake, false); + BaseMetalake insertedMetalake = + entityStore.get(metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); + assertNotNull(insertedMetalake); + assertTrue(checkMetalakeEquals(metalake, insertedMetalake)); + + // overwrite false + BaseMetalake duplicateMetalake = createMetalake(1L, "test_metalake", "this is test"); + assertThrows(RuntimeException.class, () -> entityStore.put(duplicateMetalake, false)); + + // overwrite true + BaseMetalake overittenMetalake = createMetalake(1L, "test_metalake2", "this is test2"); + entityStore.put(overittenMetalake, true); + BaseMetalake insertedMetalake1 = + entityStore.get( + overittenMetalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); + assertEquals( + 1, + entityStore.list(Namespace.empty(), BaseMetalake.class, Entity.EntityType.METALAKE).size()); + assertEquals("test_metalake2", insertedMetalake1.name()); + assertEquals("this is test2", insertedMetalake1.comment()); + } + + @Test + public void testPutAndList() throws IOException { + BaseMetalake metalake1 = createMetalake(1L, "test_metalake1", "this is test 1"); + BaseMetalake metalake2 = createMetalake(2L, "test_metalake2", "this is test 2"); + entityStore.put(metalake1, false); + entityStore.put(metalake2, false); + List metalakes = + entityStore.list(metalake1.namespace(), BaseMetalake.class, Entity.EntityType.METALAKE); + assertNotNull(metalakes); + assertEquals(2, metalakes.size()); + assertTrue(checkMetalakeEquals(metalake1, metalakes.get(0))); + assertTrue(checkMetalakeEquals(metalake2, metalakes.get(1))); + } + + @Test + public void testPutAndDelete() throws IOException { + BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); + entityStore.put(metalake, false); + entityStore.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, false); + assertThrows( + NoSuchEntityException.class, + () -> + entityStore.get( + metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class)); + } + + @Test + public void testPutAndUpdate() throws IOException { + BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); + entityStore.put(metalake, false); + + assertThrows( + RuntimeException.class, + () -> + entityStore.update( + metalake.nameIdentifier(), + BaseMetalake.class, + Entity.EntityType.METALAKE, + m -> { + BaseMetalake.Builder builder = + new BaseMetalake.Builder() + // Change the id, which is not allowed + .withId(2L) + .withName("test_metalake2") + .withComment("this is test 2") + .withProperties(new HashMap<>()) + .withAuditInfo((AuditInfo) m.auditInfo()) + .withVersion(m.getVersion()); + return builder.build(); + })); + + AuditInfo changedAuditInfo = + AuditInfo.builder().withCreator("changed_creator").withCreateTime(Instant.now()).build(); + BaseMetalake updatedMetalake = + entityStore.update( + metalake.nameIdentifier(), + BaseMetalake.class, + Entity.EntityType.METALAKE, + m -> { + BaseMetalake.Builder builder = + new BaseMetalake.Builder() + .withId(m.id()) + .withName("test_metalake2") + .withComment("this is test 2") + .withProperties(new HashMap<>()) + .withAuditInfo(changedAuditInfo) + .withVersion(m.getVersion()); + return builder.build(); + }); + BaseMetalake storedMetalake = + entityStore.get( + updatedMetalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); + assertEquals(metalake.id(), storedMetalake.id()); + assertEquals("test_metalake2", updatedMetalake.name()); + assertEquals("this is test 2", updatedMetalake.comment()); + assertEquals(changedAuditInfo.creator(), updatedMetalake.auditInfo().creator()); + } + + private static BaseMetalake createMetalake(Long id, String name, String comment) { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + return new BaseMetalake.Builder() + .withId(id) + .withName(name) + .withComment(comment) + .withProperties(new HashMap<>()) + .withAuditInfo(auditInfo) + .withVersion(SchemaVersion.V_0_1) + .build(); + } + + private static boolean checkMetalakeEquals(BaseMetalake expected, BaseMetalake actual) { + return expected.id().equals(actual.id()) + && expected.name().equals(actual.name()) + && expected.comment().equals(actual.comment()) + && expected.properties().equals(actual.properties()) + && expected.auditInfo().equals(actual.auditInfo()) + && expected.getVersion().equals(actual.getVersion()); + } + + private static void truncateAllTables() { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + try (Statement statement = connection.createStatement()) { + String query = "SHOW TABLES"; + List tableList = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(query)) { + while (rs.next()) { + tableList.add(rs.getString(1)); + } + } + for (String table : tableList) { + statement.execute("TRUNCATE TABLE " + table); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Clear table failed", e); + } + } + + private static void dropAllTables() { + try (SqlSession sqlSession = + SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { + try (Connection connection = sqlSession.getConnection()) { + try (Statement statement = connection.createStatement()) { + String query = "SHOW TABLES"; + List tableList = new ArrayList<>(); + try (ResultSet rs = statement.executeQuery(query)) { + while (rs.next()) { + tableList.add(rs.getString(1)); + } + } + for (String table : tableList) { + statement.execute("DROP TABLE " + table); + } + } + } + } catch (SQLException e) { + throw new RuntimeException("Drop table failed", e); + } + } +} From 11f4ed60f81c58d4433a6e4b1e87275f41ea2973 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 5 Feb 2024 17:38:39 +0800 Subject: [PATCH 30/34] add mysql backend ops for catalog --- LICENSE.bin | 1 - .../com/datastrato/gravitino/Configs.java | 5 - .../storage/relation/mysql/MySQLBackend.java | 235 -------------- .../mysql/mapper/MetalakeMetaMapper.java | 92 ------ .../mysql/orm/SqlSessionFactoryHelper.java | 84 ----- .../relation/mysql/orm/SqlSessions.java | 85 ----- .../storage/relation/mysql/po/MetalakePO.java | 136 -------- .../relation/mysql/utils/POConverters.java | 57 ---- .../relational/mapper/CatalogMetaMapper.java | 113 +++++++ .../storage/relational/po/CatalogPO.java | 168 ++++++++++ .../session/SqlSessionFactoryHelper.java | 2 + .../relational/utils/POConverters.java | 68 ++++ core/src/main/resources/mysql/mysql_init.sql | 16 +- .../relation/TestRelationEntityStore.java | 290 ------------------ .../session/TestSqlSessionFactoryHelper.java | 102 ------ .../mysql/session/TestSqlSessions.java | 83 ----- core/src/test/resources/h2/h2-init.sql | 17 +- 17 files changed, 382 insertions(+), 1172 deletions(-) delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java delete mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java delete mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java delete mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java delete mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java diff --git a/LICENSE.bin b/LICENSE.bin index 31de2f1c616..3456cec1422 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -411,7 +411,6 @@ Mozilla Public License Javassist - H2 Database Engine This product bundles various third-party components also placed in diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index f92ced76698..384f47628bf 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -28,11 +28,6 @@ public interface Configs { String ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY = "gravitino.entity.store.relational.jdbcPassword"; - String MYSQL_ENTITY_STORE_URL_KEY = "gravitino.entity.store.mysql.url"; - String MYSQL_ENTITY_STORE_DRIVER_NAME_KEY = "gravitino.entity.store.mysql.driverName"; - String MYSQL_ENTITY_STORE_USERNAME_KEY = "gravitino.entity.store.mysql.username"; - String MYSQL_ENTITY_STORE_PASSWORD_KEY = "gravitino.entity.store.mysql.password"; - String ENTITY_KV_ROCKSDB_BACKEND_PATH_KEY = "gravitino.entity.store.kv.rocksdbPath"; Long DEFAULT_KV_DELETE_AFTER_TIME = 604800000L; // 7 days diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java deleted file mode 100644 index 097b9c3ff82..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/MySQLBackend.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.storage.relation.mysql; - -import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.EntityAlreadyExistsException; -import com.datastrato.gravitino.HasIdentifier; -import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.AlreadyExistsException; -import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.meta.BaseMetalake; -import com.datastrato.gravitino.storage.relation.RelationBackend; -import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; -import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessionFactoryHelper; -import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessions; -import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; -import com.datastrato.gravitino.storage.relation.mysql.utils.POConverters; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Preconditions; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.apache.ibatis.session.SqlSession; - -/** - * {@link MySQLBackend} is a MySQL implementation of RelationBackend interface. If we want to use - * another relation implementation, We can just implement {@link RelationBackend} interface and use - * it in the Gravitino. - */ -public class MySQLBackend implements RelationBackend { - - /** Initialize the MySQL backend instance. */ - @Override - public void initialize(Config config) { - SqlSessionFactoryHelper.getInstance().init(config); - } - - @Override - public List list( - Namespace namespace, Entity.EntityType entityType) { - try (SqlSession session = SqlSessions.getSqlSession()) { - switch (entityType) { - case METALAKE: - List metalakePOS = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .listMetalakePOs(); - return metalakePOS != null - ? metalakePOS.stream() - .map( - metalakePO -> { - try { - return (E) POConverters.fromMetalakePO(metalakePO); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()) - : new ArrayList<>(); - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for list operation", entityType)); - } - } finally { - SqlSessions.closeSqlSession(); - } - } - - @Override - public boolean exists(NameIdentifier ident, Entity.EntityType entityType) { - try (SqlSession session = SqlSessions.getSqlSession()) { - switch (entityType) { - case METALAKE: - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - return metalakePO != null; - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for exists operation", entityType)); - } - } finally { - SqlSessions.closeSqlSession(); - } - } - - @Override - public void insert(E e, boolean overwritten) - throws EntityAlreadyExistsException { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - if (e instanceof BaseMetalake) { - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(e.nameIdentifier().name()); - if (!overwritten && metalakePO != null) { - throw new EntityAlreadyExistsException( - String.format("Metalake entity: %s already exists", e.nameIdentifier().name())); - } - - if (overwritten) { - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMetaWithUpdate(POConverters.toMetalakePO((BaseMetalake) e)); - } else { - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .insertMetalakeMeta(POConverters.toMetalakePO((BaseMetalake) e)); - } - SqlSessions.commitAndCloseSqlSession(); - } else { - SqlSessions.closeSqlSession(); - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for put operation", e.getClass())); - } - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); - } - } - } - - @Override - public E update( - NameIdentifier ident, Entity.EntityType entityType, Function updater) - throws NoSuchEntityException, AlreadyExistsException { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - switch (entityType) { - case METALAKE: - MetalakePO oldMetalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); - BaseMetalake newMetalakeEntity = (BaseMetalake) updater.apply((E) oldMetalakeEntity); - Preconditions.checkArgument( - Objects.equals(oldMetalakeEntity.id(), newMetalakeEntity.id()), - String.format( - "The updated metalake entity id: %s should same with the metalake entity id before: %s", - newMetalakeEntity.id(), oldMetalakeEntity.id())); - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .updateMetalakeMeta(POConverters.toMetalakePO(newMetalakeEntity), oldMetalakePO); - SqlSessions.commitAndCloseSqlSession(); - return (E) newMetalakeEntity; - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for update operation", entityType)); - } - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); - } - } - } - - @Override - public E get( - NameIdentifier ident, Entity.EntityType entityType) - throws NoSuchEntityException, IOException { - try (SqlSession session = SqlSessions.getSqlSession()) { - switch (entityType) { - case METALAKE: - MetalakePO metalakePO = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeMetaByName(ident.name()); - if (metalakePO == null) { - return null; - } - return (E) POConverters.fromMetalakePO(metalakePO); - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for get operation", entityType)); - } - } finally { - SqlSessions.closeSqlSession(); - } - } - - @Override - public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { - try (SqlSession session = SqlSessions.getSqlSession()) { - try { - switch (entityType) { - case METALAKE: - Long metalakeId = - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .selectMetalakeIdMetaByName(ident.name()); - if (metalakeId != null) { - // delete metalake - ((MetalakeMetaMapper) SqlSessions.getMapper(MetalakeMetaMapper.class)) - .deleteMetalakeMetaById(metalakeId); - if (cascade) { - // TODO We will cascade delete the metadata of sub-resources under the metalake - } - SqlSessions.commitAndCloseSqlSession(); - } - return true; - case CATALOG: - case SCHEMA: - case TABLE: - case FILESET: - default: - throw new IllegalArgumentException( - String.format("Unsupported entity type: %s for delete operation", entityType)); - } - } catch (Throwable t) { - SqlSessions.rollbackAndCloseSqlSession(); - throw new RuntimeException(t); - } - } - } - - @Override - public void close() throws IOException {} -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java deleted file mode 100644 index 5f863f88328..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/mapper/MetalakeMetaMapper.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relation.mysql.mapper; - -import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; -import java.util.List; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; - -public interface MetalakeMetaMapper { - String TABLE_NAME = "metalake_meta"; - - @Select( - "SELECT id, metalake_name as metalakeName, metalake_comment as metalakeComment," - + " properties, audit_info as auditInfo, schema_version as schemaVersion" - + " FROM " - + TABLE_NAME) - List listMetalakePOs(); - - @Select( - "SELECT id, metalake_name as metalakeName," - + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion" - + " FROM " - + TABLE_NAME - + " WHERE metalake_name = #{metalakeName}") - MetalakePO selectMetalakeMetaByName(@Param("metalakeName") String name); - - @Select("SELECT id FROM " + TABLE_NAME + " WHERE metalake_name = #{metalakeName}") - Long selectMetalakeIdMetaByName(@Param("metalakeName") String name); - - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" - + " VALUES(" - + " #{metalakeMeta.id}," - + " #{metalakeMeta.metalakeName}," - + " #{metalakeMeta.metalakeComment}," - + " #{metalakeMeta.properties}," - + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}" - + " )") - void insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); - - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(id, metalake_name, metalake_comment, properties, audit_info, schema_version)" - + " VALUES(" - + " #{metalakeMeta.id}," - + " #{metalakeMeta.metalakeName}," - + " #{metalakeMeta.metalakeComment}," - + " #{metalakeMeta.properties}," - + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " metalake_name = #{metalakeMeta.metalakeName}," - + " metalake_comment = #{metalakeMeta.metalakeComment}," - + " properties = #{metalakeMeta.properties}," - + " audit_info = #{metalakeMeta.auditInfo}," - + " schema_version = #{metalakeMeta.schemaVersion}") - void insertMetalakeMetaWithUpdate(@Param("metalakeMeta") MetalakePO metalakePO); - - @Update( - "UPDATE " - + TABLE_NAME - + " SET metalake_name = #{newMetalakeMeta.metalakeName}," - + " metalake_comment = #{newMetalakeMeta.metalakeComment}," - + " properties = #{newMetalakeMeta.properties}," - + " audit_info = #{newMetalakeMeta.auditInfo}," - + " schema_version = #{newMetalakeMeta.schemaVersion}" - + " WHERE id = #{oldMetalakeMeta.id}" - + " and metalake_name = #{oldMetalakeMeta.metalakeName}" - + " and metalake_comment = #{oldMetalakeMeta.metalakeComment}" - + " and properties = #{oldMetalakeMeta.properties}" - + " and audit_info = #{oldMetalakeMeta.auditInfo}" - + " and schema_version = #{oldMetalakeMeta.schemaVersion}") - void updateMetalakeMeta( - @Param("newMetalakeMeta") MetalakePO newMetalakePO, - @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); - - @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") - Integer deleteMetalakeMetaById(@Param("id") Long id); -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java deleted file mode 100644 index 67162dd761c..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessionFactoryHelper.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relation.mysql.orm; - -import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Configs; -import com.datastrato.gravitino.storage.relation.mysql.mapper.MetalakeMetaMapper; -import com.google.common.base.Preconditions; -import java.time.Duration; -import org.apache.commons.dbcp2.BasicDataSource; -import org.apache.commons.pool2.impl.BaseObjectPoolConfig; -import org.apache.ibatis.mapping.Environment; -import org.apache.ibatis.session.Configuration; -import org.apache.ibatis.session.SqlSessionFactory; -import org.apache.ibatis.session.SqlSessionFactoryBuilder; -import org.apache.ibatis.transaction.TransactionFactory; -import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; - -/** - * SqlSessionFactoryHelper maintains the MyBatis's {@link SqlSessionFactory} object, which is used - * to create the {@link org.apache.ibatis.session.SqlSession} object. It is a singleton class and - * should be initialized only once. - */ -public class SqlSessionFactoryHelper { - private static volatile SqlSessionFactory sqlSessionFactory; - private static final SqlSessionFactoryHelper INSTANCE = new SqlSessionFactoryHelper(); - - public static SqlSessionFactoryHelper getInstance() { - return INSTANCE; - } - - private SqlSessionFactoryHelper() {} - - /** - * Initialize the SqlSessionFactory object. - * - * @param config Config object to get the MySQL connection details from the config. - */ - @SuppressWarnings("deprecation") - public void init(Config config) { - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(config.getRawString(Configs.MYSQL_ENTITY_STORE_URL_KEY)); - dataSource.setDriverClassName(config.getRawString(Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)); - dataSource.setUsername(config.getRawString(Configs.MYSQL_ENTITY_STORE_USERNAME_KEY, "")); - dataSource.setPassword(config.getRawString(Configs.MYSQL_ENTITY_STORE_PASSWORD_KEY, "")); - // close the auto commit, so that need manual commit - dataSource.setDefaultAutoCommit(false); - dataSource.setMaxWaitMillis(1000L); - dataSource.setMaxTotal(20); - dataSource.setMaxIdle(5); - dataSource.setMinIdle(0); - dataSource.setLogAbandoned(true); - dataSource.setRemoveAbandonedOnBorrow(true); - dataSource.setRemoveAbandonedTimeout(60); - dataSource.setTimeBetweenEvictionRunsMillis(Duration.ofMillis(10 * 60 * 1000L).toMillis()); - dataSource.setTestOnBorrow(BaseObjectPoolConfig.DEFAULT_TEST_ON_BORROW); - dataSource.setTestWhileIdle(BaseObjectPoolConfig.DEFAULT_TEST_WHILE_IDLE); - dataSource.setMinEvictableIdleTimeMillis(1000); - dataSource.setNumTestsPerEvictionRun(BaseObjectPoolConfig.DEFAULT_NUM_TESTS_PER_EVICTION_RUN); - dataSource.setTestOnReturn(BaseObjectPoolConfig.DEFAULT_TEST_ON_RETURN); - dataSource.setSoftMinEvictableIdleTimeMillis( - BaseObjectPoolConfig.DEFAULT_SOFT_MIN_EVICTABLE_IDLE_TIME.toMillis()); - dataSource.setLifo(BaseObjectPoolConfig.DEFAULT_LIFO); - TransactionFactory transactionFactory = new JdbcTransactionFactory(); - Environment environment = new Environment("development", transactionFactory, dataSource); - Configuration configuration = new Configuration(environment); - configuration.addMapper(MetalakeMetaMapper.class); - if (sqlSessionFactory == null) { - synchronized (SqlSessionFactoryHelper.class) { - if (sqlSessionFactory == null) { - sqlSessionFactory = new SqlSessionFactoryBuilder().build(configuration); - } - } - } - } - - public SqlSessionFactory getSqlSessionFactory() { - Preconditions.checkState(sqlSessionFactory != null, "SqlSessionFactory is not initialized."); - return sqlSessionFactory; - } -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java deleted file mode 100644 index 5c3398efaa7..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/orm/SqlSessions.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relation.mysql.orm; - -import org.apache.ibatis.session.SqlSession; -import org.apache.ibatis.session.TransactionIsolationLevel; - -/** - * SqlSessions is a utility class to maintain the MyBatis's {@link SqlSession} object. It is a - * thread local class and should be used to get the {@link SqlSession} object. It also provides the - * methods to commit, rollback and close the {@link SqlSession} object. - */ -public final class SqlSessions { - private static final ThreadLocal sessions = new ThreadLocal<>(); - - private SqlSessions() {} - - /** - * Get the SqlSession object. If the SqlSession object is not present in the thread local, then - * create a new SqlSession object and set it in the thread local. - * - * @return SqlSession object from the thread local storage. - */ - public static SqlSession getSqlSession() { - SqlSession sqlSession = sessions.get(); - if (sqlSession == null) { - sqlSession = - SqlSessionFactoryHelper.getInstance() - .getSqlSessionFactory() - .openSession(TransactionIsolationLevel.READ_COMMITTED); - sessions.set(sqlSession); - return sqlSession; - } - return sqlSession; - } - - /** - * Commit the SqlSession object and close it. It also removes the SqlSession object from the - * thread local storage. - */ - public static void commitAndCloseSqlSession() { - SqlSession sqlSession = sessions.get(); - if (sqlSession != null) { - sqlSession.commit(); - sqlSession.close(); - sessions.remove(); - } - } - - /** - * Rollback the SqlSession object and close it. It also removes the SqlSession object from the - * thread local storage. - */ - public static void rollbackAndCloseSqlSession() { - SqlSession sqlSession = sessions.get(); - if (sqlSession != null) { - sqlSession.rollback(); - sqlSession.close(); - sessions.remove(); - } - } - - /** Close the SqlSession object and remove it from the thread local storage. */ - public static void closeSqlSession() { - SqlSession sqlSession = sessions.get(); - if (sqlSession != null) { - sqlSession.close(); - sessions.remove(); - } - } - - /** - * Get the Mapper object from the SqlSession object. - * - * @param className the class name of the Mapper object. - * @return the Mapper object. - * @param the type of the Mapper object. - */ - public static T getMapper(Class className) { - return (T) getSqlSession().getMapper(className); - } -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java deleted file mode 100644 index f41e82fbc4a..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/po/MetalakePO.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relation.mysql.po; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Objects; - -public class MetalakePO { - private Long id; - private String metalakeName; - private String metalakeComment; - private String properties; - private String auditInfo; - private String schemaVersion; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getMetalakeName() { - return metalakeName; - } - - public void setMetalakeName(String metalakeName) { - this.metalakeName = metalakeName; - } - - public String getMetalakeComment() { - return metalakeComment; - } - - public void setMetalakeComment(String metalakeComment) { - this.metalakeComment = metalakeComment; - } - - public String getProperties() { - return properties; - } - - public void setProperties(String properties) { - this.properties = properties; - } - - public String getAuditInfo() { - return auditInfo; - } - - public void setAuditInfo(String auditInfo) { - this.auditInfo = auditInfo; - } - - public String getSchemaVersion() { - return schemaVersion; - } - - public void setSchemaVersion(String schemaVersion) { - this.schemaVersion = schemaVersion; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof MetalakePO)) { - return false; - } - MetalakePO that = (MetalakePO) o; - return Objects.equal(getId(), that.getId()) - && Objects.equal(getMetalakeName(), that.getMetalakeName()) - && Objects.equal(getMetalakeComment(), that.getMetalakeComment()) - && Objects.equal(getProperties(), that.getProperties()) - && Objects.equal(getAuditInfo(), that.getAuditInfo()) - && Objects.equal(getSchemaVersion(), that.getSchemaVersion()); - } - - @Override - public int hashCode() { - return Objects.hashCode( - getId(), - getMetalakeName(), - getMetalakeComment(), - getProperties(), - getAuditInfo(), - getSchemaVersion()); - } - - public static class Builder { - private final MetalakePO metalakePO; - - public Builder() { - metalakePO = new MetalakePO(); - } - - public MetalakePO.Builder withId(Long id) { - metalakePO.id = id; - return this; - } - - public MetalakePO.Builder withMetalakeName(String name) { - metalakePO.metalakeName = name; - return this; - } - - public MetalakePO.Builder withMetalakeComment(String comment) { - metalakePO.metalakeComment = comment; - return this; - } - - public MetalakePO.Builder withProperties(String properties) throws JsonProcessingException { - metalakePO.properties = properties; - return this; - } - - public MetalakePO.Builder withAuditInfo(String auditInfo) throws JsonProcessingException { - metalakePO.auditInfo = auditInfo; - return this; - } - - public MetalakePO.Builder withVersion(String version) throws JsonProcessingException { - metalakePO.schemaVersion = version; - return this; - } - - public MetalakePO build() { - return metalakePO; - } - } -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java deleted file mode 100644 index d15c3a1f918..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/storage/relation/mysql/utils/POConverters.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relation.mysql.utils; - -import com.datastrato.gravitino.json.JsonUtils; -import com.datastrato.gravitino.meta.AuditInfo; -import com.datastrato.gravitino.meta.BaseMetalake; -import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relation.mysql.po.MetalakePO; -import com.fasterxml.jackson.core.JsonProcessingException; -import java.util.Map; - -/** POConverters is a utility class to convert PO to Base and vice versa. */ -public class POConverters { - private POConverters() {} - - /** - * Convert {@link BaseMetalake} to {@link MetalakePO} - * - * @param baseMetalake BaseMetalake object - * @return MetalakePO object from BaseMetalake object - * @throws JsonProcessingException - */ - public static MetalakePO toMetalakePO(BaseMetalake baseMetalake) throws JsonProcessingException { - return new MetalakePO.Builder() - .withId(baseMetalake.id()) - .withMetalakeName(baseMetalake.name()) - .withMetalakeComment(baseMetalake.comment()) - .withProperties(JsonUtils.objectMapper().writeValueAsString(baseMetalake.properties())) - .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(baseMetalake.auditInfo())) - .withVersion(JsonUtils.objectMapper().writeValueAsString(baseMetalake.getVersion())) - .build(); - } - - /** - * Convert {@link MetalakePO} to {@link BaseMetalake} - * - * @param metalakePO MetalakePO object - * @return BaseMetalake object from MetalakePO object - * @throws JsonProcessingException - */ - public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) throws JsonProcessingException { - return new BaseMetalake.Builder() - .withId(metalakePO.getId()) - .withName(metalakePO.getMetalakeName()) - .withComment(metalakePO.getMetalakeComment()) - .withProperties(JsonUtils.objectMapper().readValue(metalakePO.getProperties(), Map.class)) - .withAuditInfo( - JsonUtils.objectMapper().readValue(metalakePO.getAuditInfo(), AuditInfo.class)) - .withVersion( - JsonUtils.objectMapper().readValue(metalakePO.getSchemaVersion(), SchemaVersion.class)) - .build(); - } -} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java new file mode 100644 index 00000000000..c0112ab68e1 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -0,0 +1,113 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.mapper; + +import com.datastrato.gravitino.storage.relational.po.CatalogPO; +import java.util.List; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +public interface CatalogMetaMapper { + String TABLE_NAME = "catalog_meta"; + + @Select( + "SELECT id, catalog_name as catalogName, metalake_id as metalakeId," + + " type, provider, catalog_comment as catalogComment," + + " properties, audit_info as auditInfo" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId}") + List listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId); + + @Select( + "SELECT id FROM " + + TABLE_NAME + + " WHERE catalog_name = #{catalogName} and metalake_id = #{metalakeId}") + Long selectCatalogIdByNameAndMetalakeId( + @Param("catalogName") String name, @Param("metalakeId") Long metalakeId); + + @Select( + "SELECT id, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties," + + " audit_info as auditInfo" + + " FROM " + + TABLE_NAME + + " WHERE catalog_name = #{catalogName} and metalake_id = #{metalakeId}") + CatalogPO selectCatalogMetaByNameAndMetalakeId( + @Param("catalogName") String name, @Param("metalakeId") Long metalakeId); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(id, catalog_name, metalake_id, type, provider, catalog_comment, properties, audit_info)" + + " VALUES(" + + " #{catalogMeta.id}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}" + + " )") + void insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO); + + @Insert( + "INSERT INTO " + + TABLE_NAME + + "(id, catalog_name, metalake_id, type, provider, metalake_comment, properties, audit_info)" + + " VALUES(" + + " #{catalogMeta.id}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " catalog_name = #{catalogMeta.catalogName}," + + " metalake_id = #{catalogMeta.metalakeId}," + + " type = #{catalogMeta.type}," + + " provider = #{catalogMeta.provider}," + + " catalog_comment = #{catalogMeta.catalogComment}," + + " properties = #{catalogMeta.properties}," + + " audit_info = #{catalogMeta.auditInfo}") + void insertCatalogMetaOnDuplicateKeyUpdate(@Param("catalogMeta") CatalogPO catalogPO); + + @Update( + "UPDATE " + + TABLE_NAME + + " SET catalog_name = #{newCatalogMeta.metalakeName}," + + " metalake_id = #{newCatalogMeta.metalakeId}," + + " type = #{newCatalogMeta.type}," + + " provider = #{newCatalogMeta.provider}," + + " catalog_comment = #{newCatalogMeta.catalogComment}," + + " properties = #{newCatalogMeta.properties}," + + " audit_info = #{newCatalogMeta.auditInfo}" + + " WHERE id = #{oldCatalogMeta.id}" + + " and catalog_name = #{oldCatalogMeta.catalogName}" + + " and metalake_id = #{oldCatalogMeta.metalakeId}" + + " and type = #{oldCatalogMeta.type}" + + " and provider = #{oldCatalogMeta.provider}" + + " and catalog_comment = #{oldCatalogMeta.catalogComment}" + + " and properties = #{oldCatalogMeta.properties}" + + " and audit_info = #{oldCatalogMeta.auditInfo}") + Integer updateCatalogMeta( + @Param("newCatalogMeta") CatalogPO newCatalogPO, + @Param("oldCatalogMeta") CatalogPO oldCatalogPO); + + @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") + Integer deleteCatalogMetasById(@Param("id") Long id); + + @Delete("DELETE FROM " + TABLE_NAME + " WHERE metalake_id = #{metalakeId}") + Integer deleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java new file mode 100644 index 00000000000..feed4ecfc22 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java @@ -0,0 +1,168 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.po; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.base.Objects; + +public class CatalogPO { + private Long id; + private String catalogName; + private Long metalakeId; + private String type; + private String provider; + private String catalogComment; + private String properties; + private String auditInfo; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getCatalogName() { + return catalogName; + } + + public void setCatalogName(String catalogName) { + this.catalogName = catalogName; + } + + public Long getMetalakeId() { + return metalakeId; + } + + public void setMetalakeId(Long metalakeId) { + this.metalakeId = metalakeId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + public String getCatalogComment() { + return catalogComment; + } + + public void setCatalogComment(String catalogComment) { + this.catalogComment = catalogComment; + } + + public String getProperties() { + return properties; + } + + public void setProperties(String properties) { + this.properties = properties; + } + + public String getAuditInfo() { + return auditInfo; + } + + public void setAuditInfo(String auditInfo) { + this.auditInfo = auditInfo; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CatalogPO)) { + return false; + } + CatalogPO catalogPO = (CatalogPO) o; + return Objects.equal(getId(), catalogPO.getId()) + && Objects.equal(getCatalogName(), catalogPO.getCatalogName()) + && Objects.equal(getMetalakeId(), catalogPO.getMetalakeId()) + && Objects.equal(getType(), catalogPO.getType()) + && Objects.equal(getProvider(), catalogPO.getProvider()) + && Objects.equal(getCatalogComment(), catalogPO.getCatalogComment()) + && Objects.equal(getProperties(), catalogPO.getProperties()) + && Objects.equal(getAuditInfo(), catalogPO.getAuditInfo()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getId(), + getCatalogName(), + getMetalakeId(), + getType(), + getProvider(), + getCatalogComment(), + getProperties(), + getAuditInfo()); + } + + public static class Builder { + private final CatalogPO metalakePO; + + public Builder() { + metalakePO = new CatalogPO(); + } + + public CatalogPO.Builder withId(Long id) { + metalakePO.id = id; + return this; + } + + public CatalogPO.Builder withCatalogName(String name) { + metalakePO.catalogName = name; + return this; + } + + public CatalogPO.Builder withMetalakeId(Long metalakeId) { + metalakePO.metalakeId = metalakeId; + return this; + } + + public CatalogPO.Builder withType(String type) { + metalakePO.type = type; + return this; + } + + public CatalogPO.Builder withProvider(String provider) { + metalakePO.provider = provider; + return this; + } + + public CatalogPO.Builder withCatalogComment(String comment) { + metalakePO.catalogComment = comment; + return this; + } + + public CatalogPO.Builder withProperties(String properties) throws JsonProcessingException { + metalakePO.properties = properties; + return this; + } + + public CatalogPO.Builder withAuditInfo(String auditInfo) throws JsonProcessingException { + metalakePO.auditInfo = auditInfo; + return this; + } + + public CatalogPO build() { + return metalakePO; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index aaff7ce6130..b8dcd440115 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; +import com.datastrato.gravitino.storage.relational.mapper.CatalogMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.google.common.base.Preconditions; import java.sql.SQLException; @@ -74,6 +75,7 @@ public void init(Config config) { // Initialize the configuration Configuration configuration = new Configuration(environment); configuration.addMapper(MetalakeMetaMapper.class); + configuration.addMapper(CatalogMetaMapper.class); // Create the SqlSessionFactory object, it is a singleton object if (sqlSessionFactory == null) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 0c03faa77ba..41ff419ca32 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -5,10 +5,14 @@ package com.datastrato.gravitino.storage.relational.utils; +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relational.po.CatalogPO; import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.annotations.VisibleForTesting; @@ -125,4 +129,68 @@ public static BaseMetalake fromMetalakePO(MetalakePO metalakePO) { public static List fromMetalakePOs(List metalakePOS) { return metalakePOS.stream().map(POConverters::fromMetalakePO).collect(Collectors.toList()); } + + /** + * Convert {@link CatalogEntity} to {@link CatalogPO} + * + * @param catalogEntity CatalogEntity object to be converted + * @param metalakeId Metalake id to be associated with the catalog + * @return CatalogPO object from CatalogEntity object + */ + public static CatalogPO toCatalogPO(CatalogEntity catalogEntity, Long metalakeId) { + try { + return new CatalogPO.Builder() + .withId(catalogEntity.id()) + .withCatalogName(catalogEntity.name()) + .withMetalakeId(metalakeId) + .withType(catalogEntity.type().name()) + .withProvider(catalogEntity.getProvider()) + .withCatalogComment(catalogEntity.getComment()) + .withProperties( + JsonUtils.objectMapper().writeValueAsString(catalogEntity.getProperties())) + .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(catalogEntity.auditInfo())) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } + + /** + * Convert {@link CatalogPO} to {@link CatalogEntity} + * + * @param catalogPO CatalogPO object to be converted + * @param namespace Namespace object to be associated with the catalog + * @return CatalogEntity object from CatalogPO object + */ + public static CatalogEntity fromCatalogPO(CatalogPO catalogPO, Namespace namespace) { + try { + return CatalogEntity.builder() + .withId(catalogPO.getId()) + .withName(catalogPO.getCatalogName()) + .withNamespace(namespace) + .withType(Catalog.Type.valueOf(catalogPO.getType())) + .withProvider(catalogPO.getProvider()) + .withComment(catalogPO.getCatalogComment()) + .withProperties(JsonUtils.objectMapper().readValue(catalogPO.getProperties(), Map.class)) + .withAuditInfo( + JsonUtils.objectMapper().readValue(catalogPO.getAuditInfo(), AuditInfo.class)) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to deserialize json object:", e); + } + } + + /** + * Convert list of {@link MetalakePO} to list of {@link BaseMetalake} + * + * @param catalogPOS list of MetalakePO objects + * @param namespace Namespace object to be associated with the metalake + * @return list of BaseMetalake objects from list of MetalakePO objects + */ + public static List fromCatalogPOs( + List catalogPOS, Namespace namespace) { + return catalogPOS.stream() + .map(catalogPO -> POConverters.fromCatalogPO(catalogPO, namespace)) + .collect(Collectors.toList()); + } } diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql index 77f300810b6..d4d31b0994b 100644 --- a/core/src/main/resources/mysql/mysql_init.sql +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -15,4 +15,18 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', PRIMARY KEY (`metalake_id`), UNIQUE KEY `uk_mn_del` (`metalake_name`, `deleted_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; \ No newline at end of file +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; + +CREATE TABLE IF NOT EXISTS `catalog_meta` +( + `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', + `provider` VARCHAR(64) NOT NULL COMMENT 'catalog provider', + `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_cn_mid` (`catalog_name`, `metalake_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'catalog metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java deleted file mode 100644 index 71dd43ac9b9..00000000000 --- a/core/src/test/java/com/datastrato/gravitino/storage/relation/TestRelationEntityStore.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relation; - -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATION_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATION_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; -import static com.datastrato.gravitino.Configs.RELATION_ENTITY_STORE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Entity; -import com.datastrato.gravitino.EntityStore; -import com.datastrato.gravitino.EntityStoreFactory; -import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.exceptions.NoSuchEntityException; -import com.datastrato.gravitino.meta.AuditInfo; -import com.datastrato.gravitino.meta.BaseMetalake; -import com.datastrato.gravitino.meta.SchemaVersion; -import com.datastrato.gravitino.storage.relation.mysql.orm.SqlSessionFactoryHelper; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.UUID; -import org.apache.ibatis.session.SqlSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class TestRelationEntityStore { - private static final String MYSQL_STORE_PATH = - "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); - private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; - private static EntityStore entityStore = null; - - @BeforeAll - public static void setUp() { - File dir = new File(DB_DIR); - if (dir.exists() || !dir.isDirectory()) { - dir.delete(); - } - dir.mkdirs(); - - // Use H2 DATABASE to simulate MySQL - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATION_ENTITY_STORE); - Mockito.when(config.get(ENTITY_RELATION_STORE)).thenReturn(DEFAULT_ENTITY_RELATION_STORE); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) - .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) - .thenReturn("org.h2.Driver"); - entityStore = EntityStoreFactory.createEntityStore(config); - entityStore.initialize(config); - - // Read the ddl sql to create table - String scriptPath = "h2/h2-init.sql"; - try (SqlSession sqlSession = - SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); - Connection connection = sqlSession.getConnection(); - Statement statement = connection.createStatement()) { - URL scriptUrl = ClassLoader.getSystemResource(scriptPath); - if (scriptUrl == null) { - throw new IllegalStateException("Cannot find init sql script:" + scriptPath); - } - StringBuilder ddlBuilder = new StringBuilder(); - try (InputStreamReader inputStreamReader = - new InputStreamReader( - Files.newInputStream(Paths.get(scriptUrl.getPath())), StandardCharsets.UTF_8); - BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { - String line; - while ((line = bufferedReader.readLine()) != null) { - ddlBuilder.append(line).append("\n"); - } - } - statement.execute(ddlBuilder.toString()); - } catch (Exception e) { - throw new IllegalStateException("Create tables failed", e); - } - } - - @AfterEach - public void destroy() { - truncateAllTables(); - } - - @AfterAll - public static void tearDown() throws IOException { - dropAllTables(); - entityStore.close(); - File dir = new File(DB_DIR); - if (dir.exists()) { - dir.delete(); - } - } - - @Test - public void testPutAndGet() throws IOException { - BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); - entityStore.put(metalake, false); - BaseMetalake insertedMetalake = - entityStore.get(metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); - assertNotNull(insertedMetalake); - assertTrue(checkMetalakeEquals(metalake, insertedMetalake)); - - // overwrite false - BaseMetalake duplicateMetalake = createMetalake(1L, "test_metalake", "this is test"); - assertThrows(RuntimeException.class, () -> entityStore.put(duplicateMetalake, false)); - - // overwrite true - BaseMetalake overittenMetalake = createMetalake(1L, "test_metalake2", "this is test2"); - entityStore.put(overittenMetalake, true); - BaseMetalake insertedMetalake1 = - entityStore.get( - overittenMetalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); - assertEquals( - 1, - entityStore.list(Namespace.empty(), BaseMetalake.class, Entity.EntityType.METALAKE).size()); - assertEquals("test_metalake2", insertedMetalake1.name()); - assertEquals("this is test2", insertedMetalake1.comment()); - } - - @Test - public void testPutAndList() throws IOException { - BaseMetalake metalake1 = createMetalake(1L, "test_metalake1", "this is test 1"); - BaseMetalake metalake2 = createMetalake(2L, "test_metalake2", "this is test 2"); - entityStore.put(metalake1, false); - entityStore.put(metalake2, false); - List metalakes = - entityStore.list(metalake1.namespace(), BaseMetalake.class, Entity.EntityType.METALAKE); - assertNotNull(metalakes); - assertEquals(2, metalakes.size()); - assertTrue(checkMetalakeEquals(metalake1, metalakes.get(0))); - assertTrue(checkMetalakeEquals(metalake2, metalakes.get(1))); - } - - @Test - public void testPutAndDelete() throws IOException { - BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); - entityStore.put(metalake, false); - entityStore.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, false); - assertThrows( - NoSuchEntityException.class, - () -> - entityStore.get( - metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class)); - } - - @Test - public void testPutAndUpdate() throws IOException { - BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); - entityStore.put(metalake, false); - - assertThrows( - RuntimeException.class, - () -> - entityStore.update( - metalake.nameIdentifier(), - BaseMetalake.class, - Entity.EntityType.METALAKE, - m -> { - BaseMetalake.Builder builder = - new BaseMetalake.Builder() - // Change the id, which is not allowed - .withId(2L) - .withName("test_metalake2") - .withComment("this is test 2") - .withProperties(new HashMap<>()) - .withAuditInfo((AuditInfo) m.auditInfo()) - .withVersion(m.getVersion()); - return builder.build(); - })); - - AuditInfo changedAuditInfo = - AuditInfo.builder().withCreator("changed_creator").withCreateTime(Instant.now()).build(); - BaseMetalake updatedMetalake = - entityStore.update( - metalake.nameIdentifier(), - BaseMetalake.class, - Entity.EntityType.METALAKE, - m -> { - BaseMetalake.Builder builder = - new BaseMetalake.Builder() - .withId(m.id()) - .withName("test_metalake2") - .withComment("this is test 2") - .withProperties(new HashMap<>()) - .withAuditInfo(changedAuditInfo) - .withVersion(m.getVersion()); - return builder.build(); - }); - BaseMetalake storedMetalake = - entityStore.get( - updatedMetalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class); - assertEquals(metalake.id(), storedMetalake.id()); - assertEquals("test_metalake2", updatedMetalake.name()); - assertEquals("this is test 2", updatedMetalake.comment()); - assertEquals(changedAuditInfo.creator(), updatedMetalake.auditInfo().creator()); - } - - private static BaseMetalake createMetalake(Long id, String name, String comment) { - AuditInfo auditInfo = - AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - return new BaseMetalake.Builder() - .withId(id) - .withName(name) - .withComment(comment) - .withProperties(new HashMap<>()) - .withAuditInfo(auditInfo) - .withVersion(SchemaVersion.V_0_1) - .build(); - } - - private static boolean checkMetalakeEquals(BaseMetalake expected, BaseMetalake actual) { - return expected.id().equals(actual.id()) - && expected.name().equals(actual.name()) - && expected.comment().equals(actual.comment()) - && expected.properties().equals(actual.properties()) - && expected.auditInfo().equals(actual.auditInfo()) - && expected.getVersion().equals(actual.getVersion()); - } - - private static void truncateAllTables() { - try (SqlSession sqlSession = - SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { - try (Connection connection = sqlSession.getConnection()) { - try (Statement statement = connection.createStatement()) { - String query = "SHOW TABLES"; - List tableList = new ArrayList<>(); - try (ResultSet rs = statement.executeQuery(query)) { - while (rs.next()) { - tableList.add(rs.getString(1)); - } - } - for (String table : tableList) { - statement.execute("TRUNCATE TABLE " + table); - } - } - } - } catch (SQLException e) { - throw new RuntimeException("Clear table failed", e); - } - } - - private static void dropAllTables() { - try (SqlSession sqlSession = - SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { - try (Connection connection = sqlSession.getConnection()) { - try (Statement statement = connection.createStatement()) { - String query = "SHOW TABLES"; - List tableList = new ArrayList<>(); - try (ResultSet rs = statement.executeQuery(query)) { - while (rs.next()) { - tableList.add(rs.getString(1)); - } - } - for (String table : tableList) { - statement.execute("DROP TABLE " + table); - } - } - } - } catch (SQLException e) { - throw new RuntimeException("Drop table failed", e); - } - } -} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java deleted file mode 100644 index 1ebeac5c2f0..00000000000 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessionFactoryHelper.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relational.mysql.session; - -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; -import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.datastrato.gravitino.Config; -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; -import java.util.UUID; -import org.apache.commons.dbcp2.BasicDataSource; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class TestSqlSessionFactoryHelper { - private static final String MYSQL_STORE_PATH = - "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); - private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; - - private static Config config; - - @BeforeAll - public static void setUp() { - File dir = new File(DB_DIR); - if (dir.exists() || !dir.isDirectory()) { - dir.delete(); - } - dir.mkdirs(); - - config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) - .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) - .thenReturn("org.h2.Driver"); - } - - @BeforeEach - public void init() { - SqlSessionFactoryHelper.setSqlSessionFactory(null); - } - - @AfterEach - public void cleanUp() { - SqlSessionFactoryHelper.setSqlSessionFactory(null); - } - - @AfterAll - public static void tearDown() throws IOException { - File dir = new File(DB_DIR); - if (dir.exists()) { - dir.delete(); - } - } - - @Test - public void testGetInstance() { - SqlSessionFactoryHelper instance = SqlSessionFactoryHelper.getInstance(); - assertNotNull(instance); - } - - @Test - public void testInit() throws SQLException { - SqlSessionFactoryHelper.getInstance().init(config); - assertNotNull(SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); - BasicDataSource dataSource = - (BasicDataSource) - SqlSessionFactoryHelper.getInstance() - .getSqlSessionFactory() - .getConfiguration() - .getEnvironment() - .getDataSource(); - assertEquals("org.h2.Driver", dataSource.getDriverClassName()); - assertEquals(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY), dataSource.getUrl()); - } - - @Test - public void testGetSqlSessionFactoryWithoutInit() { - assertThrows( - IllegalStateException.class, - () -> SqlSessionFactoryHelper.getInstance().getSqlSessionFactory()); - } -} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java deleted file mode 100644 index 57c4a39e606..00000000000 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/mysql/session/TestSqlSessions.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.storage.relational.mysql.session; - -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_STORE; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_DRIVER_NAME_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_URL_KEY; -import static com.datastrato.gravitino.Configs.MYSQL_ENTITY_STORE_USERNAME_KEY; -import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -import com.datastrato.gravitino.Config; -import java.io.File; -import java.io.IOException; -import java.util.UUID; -import org.apache.ibatis.session.SqlSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class TestSqlSessions { - private static final String MYSQL_STORE_PATH = - "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); - private static final String DB_DIR = MYSQL_STORE_PATH + "/testdb"; - - @BeforeAll - public static void setUp() { - File dir = new File(DB_DIR); - if (dir.exists() || !dir.isDirectory()) { - dir.delete(); - } - dir.mkdirs(); - - Config config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); - Mockito.when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_URL_KEY)) - .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", DB_DIR)); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_USERNAME_KEY)).thenReturn("sa"); - Mockito.when(config.getRawString(MYSQL_ENTITY_STORE_DRIVER_NAME_KEY)) - .thenReturn("org.h2.Driver"); - SqlSessionFactoryHelper.getInstance().init(config); - } - - @AfterAll - public static void tearDown() throws IOException { - File dir = new File(DB_DIR); - if (dir.exists()) { - dir.delete(); - } - } - - @Test - public void testOpenAndCloseSqlSession() { - SqlSession session = SqlSessions.getSqlSession(); - assertNotNull(session); - SqlSessions.closeSqlSession(); - assertNull(SqlSessions.getSessions().get()); - } - - @Test - public void testOpenAndCommitAndCloseSqlSession() { - SqlSession session = SqlSessions.getSqlSession(); - assertNotNull(session); - SqlSessions.commitAndCloseSqlSession(); - assertNull(SqlSessions.getSessions().get()); - } - - @Test - public void testOpenAndRollbackAndCloseSqlSession() { - SqlSession session = SqlSessions.getSqlSession(); - assertNotNull(session); - SqlSessions.rollbackAndCloseSqlSession(); - assertNull(SqlSessions.getSessions().get()); - } -} diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index 74d00874577..6107ac7e105 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -15,4 +15,19 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', PRIMARY KEY (metalake_id), CONSTRAINT uk_mn_del UNIQUE (metalake_name, deleted_at) -) ENGINE = InnoDB; \ No newline at end of file +) ENGINE = InnoDB; + + +CREATE TABLE IF NOT EXISTS `catalog_meta` +( + `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', + `provider` VARCHAR(64) NOT NULL COMMENT 'catalog provider', + `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', + PRIMARY KEY (id), + CONSTRAINT uk_cn_mid UNIQUE (catalog_name, metalake_id) +) ENGINE=InnoDB; \ No newline at end of file From bb5334e8031c6e10bf86db65a02a74bcdfc0c3ac Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 26 Feb 2024 15:17:36 +0800 Subject: [PATCH 31/34] refactor the catalog code --- .../storage/relational/JDBCBackend.java | 15 +- .../relational/mapper/CatalogMetaMapper.java | 101 ++++++--- .../relational/mapper/MetalakeMetaMapper.java | 19 +- .../storage/relational/po/CatalogPO.java | 87 +++---- .../service/CatalogMetaService.java | 194 ++++++++++++++++ .../service/MetalakeMetaService.java | 35 ++- .../relational/utils/POConverters.java | 70 +++++- core/src/main/resources/mysql/mysql_init.sql | 12 +- .../relational/TestRelationalEntityStore.java | 214 +++++++++++++++++- .../relational/utils/TestPOConverters.java | 147 +++++++++++- core/src/test/resources/h2/h2-init.sql | 9 +- 11 files changed, 784 insertions(+), 119 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 1695af21779..c62676293d4 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -6,6 +6,7 @@ package com.datastrato.gravitino.storage.relational; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; @@ -15,6 +16,8 @@ import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import java.io.IOException; @@ -41,6 +44,8 @@ public List list( switch (entityType) { case METALAKE: return (List) MetalakeMetaService.getInstance().listMetalakes(); + case CATALOG: + return (List) CatalogMetaService.getInstance().listCatalogsByNamespace(namespace); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for list operation", entityType); @@ -62,6 +67,8 @@ public void insert(E e, boolean overwritten) throws EntityAlreadyExistsException { if (e instanceof BaseMetalake) { MetalakeMetaService.getInstance().insertMetalake((BaseMetalake) e, overwritten); + } else if (e instanceof CatalogEntity) { + CatalogMetaService.getInstance().insertCatalog((CatalogEntity) e, overwritten); } else { throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for insert operation", e.getClass()); @@ -75,6 +82,8 @@ public E update( switch (entityType) { case METALAKE: return (E) MetalakeMetaService.getInstance().updateMetalake(ident, updater); + case CATALOG: + return (E) CatalogMetaService.getInstance().updateCatalog(ident, updater); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for update operation", entityType); @@ -86,7 +95,9 @@ public E get( NameIdentifier ident, Entity.EntityType entityType) throws NoSuchEntityException { switch (entityType) { case METALAKE: - return (E) MetalakeMetaService.getInstance().getMetalakeByIdent(ident); + return (E) MetalakeMetaService.getInstance().getMetalakeByIdentifier(ident); + case CATALOG: + return (E) CatalogMetaService.getInstance().getCatalogByIdentifier(ident); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for get operation", entityType); @@ -98,6 +109,8 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea switch (entityType) { case METALAKE: return MetalakeMetaService.getInstance().deleteMetalake(ident, cascade); + case CATALOG: + return CatalogMetaService.getInstance().deleteCatalog(ident, cascade); default: throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for delete operation", entityType); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java index c0112ab68e1..69731e0d393 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -7,7 +7,6 @@ import com.datastrato.gravitino.storage.relational.po.CatalogPO; import java.util.List; -import org.apache.ibatis.annotations.Delete; import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -17,61 +16,74 @@ public interface CatalogMetaMapper { String TABLE_NAME = "catalog_meta"; @Select( - "SELECT id, catalog_name as catalogName, metalake_id as metalakeId," - + " type, provider, catalog_comment as catalogComment," - + " properties, audit_info as auditInfo" + "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + " FROM " + TABLE_NAME - + " WHERE metalake_id = #{metalakeId}") + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") List listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId); @Select( - "SELECT id FROM " + "SELECT catalog_id as catalogId FROM " + TABLE_NAME - + " WHERE catalog_name = #{catalogName} and metalake_id = #{metalakeId}") - Long selectCatalogIdByNameAndMetalakeId( - @Param("catalogName") String name, @Param("metalakeId") Long metalakeId); + + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0") + Long selectCatalogIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); @Select( - "SELECT id, catalog_name as catalogName," + "SELECT catalog_id as catalogId, catalog_name as catalogName," + " metalake_id as metalakeId, type, provider," - + " catalog_comment as catalogComment, properties," - + " audit_info as auditInfo" + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + " FROM " + TABLE_NAME - + " WHERE catalog_name = #{catalogName} and metalake_id = #{metalakeId}") - CatalogPO selectCatalogMetaByNameAndMetalakeId( - @Param("catalogName") String name, @Param("metalakeId") Long metalakeId); + + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0") + CatalogPO selectCatalogMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); @Insert( "INSERT INTO " + TABLE_NAME - + "(id, catalog_name, metalake_id, type, provider, catalog_comment, properties, audit_info)" + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + " VALUES(" - + " #{catalogMeta.id}," + + " #{catalogMeta.catalogId}," + " #{catalogMeta.catalogName}," + " #{catalogMeta.metalakeId}," + " #{catalogMeta.type}," + " #{catalogMeta.provider}," + " #{catalogMeta.catalogComment}," + " #{catalogMeta.properties}," - + " #{catalogMeta.auditInfo}" + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + " )") void insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO); @Insert( "INSERT INTO " + TABLE_NAME - + "(id, catalog_name, metalake_id, type, provider, metalake_comment, properties, audit_info)" + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + " VALUES(" - + " #{catalogMeta.id}," + + " #{catalogMeta.catalogId}," + " #{catalogMeta.catalogName}," + " #{catalogMeta.metalakeId}," + " #{catalogMeta.type}," + " #{catalogMeta.provider}," + " #{catalogMeta.catalogComment}," + " #{catalogMeta.properties}," - + " #{catalogMeta.auditInfo}" + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + " )" + " ON DUPLICATE KEY UPDATE" + " catalog_name = #{catalogMeta.catalogName}," @@ -80,34 +92,51 @@ CatalogPO selectCatalogMetaByNameAndMetalakeId( + " provider = #{catalogMeta.provider}," + " catalog_comment = #{catalogMeta.catalogComment}," + " properties = #{catalogMeta.properties}," - + " audit_info = #{catalogMeta.auditInfo}") + + " audit_info = #{catalogMeta.auditInfo}," + + " current_version = #{catalogMeta.currentVersion}," + + " last_version = #{catalogMeta.lastVersion}," + + " deleted_at = #{catalogMeta.deletedAt}") void insertCatalogMetaOnDuplicateKeyUpdate(@Param("catalogMeta") CatalogPO catalogPO); @Update( "UPDATE " + TABLE_NAME - + " SET catalog_name = #{newCatalogMeta.metalakeName}," + + " SET catalog_name = #{newCatalogMeta.catalogName}," + " metalake_id = #{newCatalogMeta.metalakeId}," + " type = #{newCatalogMeta.type}," + " provider = #{newCatalogMeta.provider}," + " catalog_comment = #{newCatalogMeta.catalogComment}," + " properties = #{newCatalogMeta.properties}," - + " audit_info = #{newCatalogMeta.auditInfo}" - + " WHERE id = #{oldCatalogMeta.id}" - + " and catalog_name = #{oldCatalogMeta.catalogName}" - + " and metalake_id = #{oldCatalogMeta.metalakeId}" - + " and type = #{oldCatalogMeta.type}" - + " and provider = #{oldCatalogMeta.provider}" - + " and catalog_comment = #{oldCatalogMeta.catalogComment}" - + " and properties = #{oldCatalogMeta.properties}" - + " and audit_info = #{oldCatalogMeta.auditInfo}") + + " audit_info = #{newCatalogMeta.auditInfo}," + + " current_version = #{newCatalogMeta.currentVersion}," + + " last_version = #{newCatalogMeta.lastVersion}," + + " deleted_at = #{newCatalogMeta.deletedAt}" + + " WHERE catalog_id = #{oldCatalogMeta.catalogId}" + + " AND catalog_name = #{oldCatalogMeta.catalogName}" + + " AND metalake_id = #{oldCatalogMeta.metalakeId}" + + " AND type = #{oldCatalogMeta.type}" + + " AND provider = #{oldCatalogMeta.provider}" + + " AND catalog_comment = #{oldCatalogMeta.catalogComment}" + + " AND properties = #{oldCatalogMeta.properties}" + + " AND audit_info = #{oldCatalogMeta.auditInfo}" + + " AND current_version = #{oldCatalogMeta.currentVersion}" + + " AND last_version = #{oldCatalogMeta.lastVersion}" + + " AND deleted_at = 0") Integer updateCatalogMeta( @Param("newCatalogMeta") CatalogPO newCatalogPO, @Param("oldCatalogMeta") CatalogPO oldCatalogPO); - @Delete("DELETE FROM " + TABLE_NAME + " WHERE id = #{id}") - Integer deleteCatalogMetasById(@Param("id") Long id); + @Update( + "UPDATE " + + TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP()" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + Integer softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId); - @Delete("DELETE FROM " + TABLE_NAME + " WHERE metalake_id = #{metalakeId}") - Integer deleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId); + @Update( + "UPDATE " + + TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP()" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + Integer softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index b731b718442..2336a6761a6 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -107,14 +107,14 @@ public interface MetalakeMetaMapper { + " current_version = #{newMetalakeMeta.currentVersion}," + " last_version = #{newMetalakeMeta.lastVersion}" + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" - + " and metalake_name = #{oldMetalakeMeta.metalakeName}" - + " and metalake_comment = #{oldMetalakeMeta.metalakeComment}" - + " and properties = #{oldMetalakeMeta.properties}" - + " and audit_info = #{oldMetalakeMeta.auditInfo}" - + " and schema_version = #{oldMetalakeMeta.schemaVersion}" - + " and current_version = #{oldMetalakeMeta.currentVersion}" - + " and last_version = #{oldMetalakeMeta.lastVersion}" - + " and deleted_at = 0") + + " AND metalake_name = #{oldMetalakeMeta.metalakeName}" + + " AND metalake_comment = #{oldMetalakeMeta.metalakeComment}" + + " AND properties = #{oldMetalakeMeta.properties}" + + " AND audit_info = #{oldMetalakeMeta.auditInfo}" + + " AND schema_version = #{oldMetalakeMeta.schemaVersion}" + + " AND current_version = #{oldMetalakeMeta.currentVersion}" + + " AND last_version = #{oldMetalakeMeta.lastVersion}" + + " AND deleted_at = 0") Integer updateMetalakeMeta( @Param("newMetalakeMeta") MetalakePO newMetalakePO, @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); @@ -122,6 +122,7 @@ Integer updateMetalakeMeta( @Update( "UPDATE " + TABLE_NAME - + " SET deleted_at = UNIX_TIMESTAMP() WHERE metalake_id = #{metalakeId}") + + " SET deleted_at = UNIX_TIMESTAMP()" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") Integer softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java index feed4ecfc22..10f7df52c06 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/CatalogPO.java @@ -5,11 +5,10 @@ package com.datastrato.gravitino.storage.relational.po; -import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Objects; public class CatalogPO { - private Long id; + private Long catalogId; private String catalogName; private Long metalakeId; private String type; @@ -17,69 +16,52 @@ public class CatalogPO { private String catalogComment; private String properties; private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; + public Long getCatalogId() { + return catalogId; } public String getCatalogName() { return catalogName; } - public void setCatalogName(String catalogName) { - this.catalogName = catalogName; - } - public Long getMetalakeId() { return metalakeId; } - public void setMetalakeId(Long metalakeId) { - this.metalakeId = metalakeId; - } - public String getType() { return type; } - public void setType(String type) { - this.type = type; - } - public String getProvider() { return provider; } - public void setProvider(String provider) { - this.provider = provider; - } - public String getCatalogComment() { return catalogComment; } - public void setCatalogComment(String catalogComment) { - this.catalogComment = catalogComment; - } - public String getProperties() { return properties; } - public void setProperties(String properties) { - this.properties = properties; - } - public String getAuditInfo() { return auditInfo; } - public void setAuditInfo(String auditInfo) { - this.auditInfo = auditInfo; + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; } @Override @@ -91,27 +73,33 @@ public boolean equals(Object o) { return false; } CatalogPO catalogPO = (CatalogPO) o; - return Objects.equal(getId(), catalogPO.getId()) + return Objects.equal(getCatalogId(), catalogPO.getCatalogId()) && Objects.equal(getCatalogName(), catalogPO.getCatalogName()) && Objects.equal(getMetalakeId(), catalogPO.getMetalakeId()) && Objects.equal(getType(), catalogPO.getType()) && Objects.equal(getProvider(), catalogPO.getProvider()) && Objects.equal(getCatalogComment(), catalogPO.getCatalogComment()) && Objects.equal(getProperties(), catalogPO.getProperties()) - && Objects.equal(getAuditInfo(), catalogPO.getAuditInfo()); + && Objects.equal(getAuditInfo(), catalogPO.getAuditInfo()) + && Objects.equal(getCurrentVersion(), catalogPO.getCurrentVersion()) + && Objects.equal(getLastVersion(), catalogPO.getLastVersion()) + && Objects.equal(getDeletedAt(), catalogPO.getDeletedAt()); } @Override public int hashCode() { return Objects.hashCode( - getId(), + getCatalogId(), getCatalogName(), getMetalakeId(), getType(), getProvider(), getCatalogComment(), getProperties(), - getAuditInfo()); + getAuditInfo(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); } public static class Builder { @@ -121,8 +109,8 @@ public Builder() { metalakePO = new CatalogPO(); } - public CatalogPO.Builder withId(Long id) { - metalakePO.id = id; + public CatalogPO.Builder withCatalogId(Long catalogId) { + metalakePO.catalogId = catalogId; return this; } @@ -151,16 +139,31 @@ public CatalogPO.Builder withCatalogComment(String comment) { return this; } - public CatalogPO.Builder withProperties(String properties) throws JsonProcessingException { + public CatalogPO.Builder withProperties(String properties) { metalakePO.properties = properties; return this; } - public CatalogPO.Builder withAuditInfo(String auditInfo) throws JsonProcessingException { + public CatalogPO.Builder withAuditInfo(String auditInfo) { metalakePO.auditInfo = auditInfo; return this; } + public CatalogPO.Builder withCurrentVersion(Long currentVersion) { + metalakePO.currentVersion = currentVersion; + return this; + } + + public CatalogPO.Builder withLastVersion(Long lastVersion) { + metalakePO.lastVersion = lastVersion; + return this; + } + + public CatalogPO.Builder withDeletedAt(Long deletedAt) { + metalakePO.deletedAt = deletedAt; + return this; + } + public CatalogPO build() { return metalakePO; } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java new file mode 100644 index 00000000000..9239e20da30 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -0,0 +1,194 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.storage.relational.mapper.CatalogMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import com.datastrato.gravitino.storage.relational.po.CatalogPO; +import com.datastrato.gravitino.storage.relational.utils.POConverters; +import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +public class CatalogMetaService { + private static final CatalogMetaService INSTANCE = new CatalogMetaService(); + + public static CatalogMetaService getInstance() { + return INSTANCE; + } + + private CatalogMetaService() {} + + public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) { + checkCatalogNamespaceWithIdentifier(identifier); + String metalakeName = identifier.namespace().level(0); + String catalogName = identifier.name(); + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); + if (metalakeId == null) { + throw new NoSuchEntityException("No such an entity: %s", identifier.namespace().toString()); + } + CatalogPO catalogPO = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, + mapper -> mapper.selectCatalogMetaByMetalakeIdAndName(metalakeId, catalogName)); + if (catalogPO == null) { + throw new NoSuchEntityException("No such entity: %s", identifier.toString()); + } + return POConverters.fromCatalogPO(catalogPO, identifier.namespace()); + } + + public List listCatalogsByNamespace(Namespace namespace) { + checkCatalogNamespace(namespace); + String metalakeName = namespace.level(0); + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); + if (metalakeId == null) { + throw new NoSuchEntityException("No such an entity: %s", namespace.toString()); + } + List catalogPOS = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByMetalakeId(metalakeId)); + return POConverters.fromCatalogPOs(catalogPOS, namespace); + } + + public void insertCatalog(CatalogEntity catalogEntity, boolean overwrite) { + try { + checkCatalogNamespaceWithIdentifier(catalogEntity.nameIdentifier()); + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, + mapper -> mapper.selectMetalakeIdMetaByName(catalogEntity.namespace().level(0))); + if (metalakeId == null) { + throw new NoSuchEntityException( + "No such an entity: %s", catalogEntity.namespace().toString()); + } + SessionUtils.doWithCommit( + CatalogMetaMapper.class, + mapper -> { + CatalogPO po = POConverters.initializeCatalogPOWithVersion(catalogEntity, metalakeId); + if (overwrite) { + mapper.insertCatalogMetaOnDuplicateKeyUpdate(po); + } else { + mapper.insertCatalogMeta(po); + } + }); + } catch (RuntimeException re) { + if (re.getCause() != null + && re.getCause().getCause() != null + && re.getCause().getCause() instanceof SQLIntegrityConstraintViolationException) { + // TODO We should make more fine-grained exception judgments + // Usually throwing `SQLIntegrityConstraintViolationException` means that + // SQL violates the constraints of `primary key` and `unique key`. + // We simply think that the entity already exists at this time. + throw new EntityAlreadyExistsException( + String.format( + "Catalog entity: %s already exists", catalogEntity.nameIdentifier().name())); + } + throw re; + } + } + + public CatalogEntity updateCatalog( + NameIdentifier identifier, Function updater) throws IOException { + checkCatalogNamespaceWithIdentifier(identifier); + String metalakeName = identifier.namespace().level(0); + String catalogName = identifier.name(); + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); + if (metalakeId == null) { + throw new NoSuchEntityException("No such an entity: %s", identifier.namespace().toString()); + } + + CatalogPO oldCatalogPO = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, + mapper -> mapper.selectCatalogMetaByMetalakeIdAndName(metalakeId, catalogName)); + if (oldCatalogPO == null) { + throw new NoSuchEntityException("No such an entity: %s", identifier.toString()); + } + + CatalogEntity oldCatalogEntity = + POConverters.fromCatalogPO(oldCatalogPO, identifier.namespace()); + CatalogEntity newEntity = (CatalogEntity) updater.apply((E) oldCatalogEntity); + Preconditions.checkArgument( + Objects.equals(oldCatalogEntity.id(), newEntity.id()), + "The updated catalog entity id: %s should be same with the catalog entity id before: %s", + newEntity.id(), + oldCatalogEntity.id()); + + Integer updateResult = + SessionUtils.doWithCommitAndFetchResult( + CatalogMetaMapper.class, + mapper -> + mapper.updateCatalogMeta( + POConverters.updateCatalogPOWithVersion(oldCatalogPO, newEntity, metalakeId), + oldCatalogPO)); + + if (updateResult > 0) { + return newEntity; + } else { + throw new IOException("Failed to update the entity: " + identifier); + } + } + + public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { + checkCatalogNamespaceWithIdentifier(identifier); + String metalakeName = identifier.namespace().level(0); + String catalogName = identifier.name(); + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); + if (metalakeId == null) { + throw new NoSuchEntityException("No such an entity: %s", identifier.namespace().toString()); + } + Long catalogId = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, + mapper -> mapper.selectCatalogIdByMetalakeIdAndName(metalakeId, catalogName)); + if (catalogId != null) { + if (cascade) { + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + CatalogMetaMapper.class, + mapper -> mapper.softDeleteCatalogMetasByCatalogId(catalogId)), + () -> { + // TODO We will cascade delete the metadata of sub-resources under the catalog + }); + } else { + // TODO Check whether the sub-resources are empty. If the sub-resources are not empty, + // deletion is not allowed. + SessionUtils.doWithCommit( + CatalogMetaMapper.class, mapper -> mapper.softDeleteCatalogMetasByCatalogId(catalogId)); + } + } + return true; + } + + private void checkCatalogNamespaceWithIdentifier(NameIdentifier identifier) { + Preconditions.checkArgument( + identifier.hasNamespace() && identifier.namespace().levels().length == 1, + "Only support one level namespace"); + } + + private void checkCatalogNamespace(Namespace namespace) { + Preconditions.checkArgument(namespace.levels().length == 1, "Only support one level namespace"); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 9cb1418e444..1b43d41355c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -9,8 +9,12 @@ import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.exceptions.NonEmptyEntityException; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.storage.relational.mapper.CatalogMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.datastrato.gravitino.storage.relational.utils.POConverters; @@ -38,12 +42,12 @@ public List listMetalakes() { return POConverters.fromMetalakePOs(metalakePOS); } - public BaseMetalake getMetalakeByIdent(NameIdentifier ident) { + public BaseMetalake getMetalakeByIdentifier(NameIdentifier ident) { MetalakePO metalakePO = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); if (metalakePO == null) { - throw new NoSuchEntityException("No such entity: %s", ident.toString()); + throw new NoSuchEntityException("No such an entity: %s", ident.toString()); } return POConverters.fromMetalakePO(metalakePO); } @@ -82,7 +86,7 @@ public BaseMetalake updateMetalake( SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); if (oldMetalakePO == null) { - throw new NoSuchEntityException("No such entity: %s", ident.toString()); + throw new NoSuchEntityException("No such an entity: %s", ident.toString()); } BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); @@ -107,25 +111,36 @@ public BaseMetalake updateMetalake( } public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { - Long metalakeId = + MetalakePO metalakePO = SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(ident.name())); - if (metalakeId != null) { + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); + if (metalakePO != null) { if (cascade) { SessionUtils.doMultipleWithCommit( () -> SessionUtils.doWithoutCommit( MetalakeMetaMapper.class, - mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)), + mapper -> + mapper.softDeleteMetalakeMetaByMetalakeId(metalakePO.getMetalakeId())), + () -> + SessionUtils.doWithoutCommit( + CatalogMetaMapper.class, + mapper -> + mapper.softDeleteCatalogMetasByMetalakeId(metalakePO.getMetalakeId())), () -> { // TODO We will cascade delete the metadata of sub-resources under the metalake }); } else { - // TODO Check whether the sub-resources are empty. If the sub-resources are not empty, - // deletion is not allowed. + List catalogEntities = + CatalogMetaService.getInstance() + .listCatalogsByNamespace(Namespace.ofCatalog(metalakePO.getMetalakeName())); + if (!catalogEntities.isEmpty()) { + throw new NonEmptyEntityException( + "Entity %s has sub-entities, you should remove sub-entities first", ident); + } SessionUtils.doWithCommit( MetalakeMetaMapper.class, - mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)); + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakePO.getMetalakeId())); } } return true; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 41ff419ca32..a19ea0eacaf 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -137,24 +137,77 @@ public static List fromMetalakePOs(List metalakePOS) { * @param metalakeId Metalake id to be associated with the catalog * @return CatalogPO object from CatalogEntity object */ - public static CatalogPO toCatalogPO(CatalogEntity catalogEntity, Long metalakeId) { + @VisibleForTesting + static CatalogPO toCatalogPO(CatalogEntity catalogEntity, Long metalakeId) { try { return new CatalogPO.Builder() - .withId(catalogEntity.id()) + .withCatalogId(catalogEntity.id()) .withCatalogName(catalogEntity.name()) .withMetalakeId(metalakeId) - .withType(catalogEntity.type().name()) + .withType(catalogEntity.getType().name()) .withProvider(catalogEntity.getProvider()) .withCatalogComment(catalogEntity.getComment()) .withProperties( - JsonUtils.objectMapper().writeValueAsString(catalogEntity.getProperties())) - .withAuditInfo(JsonUtils.objectMapper().writeValueAsString(catalogEntity.auditInfo())) + JsonUtils.anyFieldMapper().writeValueAsString(catalogEntity.getProperties())) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(catalogEntity.auditInfo())) .build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to serialize json object:", e); } } + /** + * Initialize CatalogPO + * + * @param catalogEntity CatalogEntity object + * @return CatalogPO object with version initialized + */ + public static CatalogPO initializeCatalogPOWithVersion( + CatalogEntity catalogEntity, Long metalakeId) { + CatalogPO catalogPO = toCatalogPO(catalogEntity, metalakeId); + return new CatalogPO.Builder() + .withCatalogId(catalogPO.getCatalogId()) + .withCatalogName(catalogPO.getCatalogName()) + .withMetalakeId(metalakeId) + .withType(catalogPO.getType()) + .withProvider(catalogPO.getProvider()) + .withCatalogComment(catalogPO.getCatalogComment()) + .withProperties(catalogPO.getProperties()) + .withAuditInfo(catalogPO.getAuditInfo()) + .withCurrentVersion(1L) + .withLastVersion(1L) + .withDeletedAt(0L) + .build(); + } + + /** + * Update CatalogPO version + * + * @param oldCatalogPO the old CatalogPO object + * @param newCatalog the new CatalogEntity object + * @return CatalogPO object with updated version + */ + public static CatalogPO updateCatalogPOWithVersion( + CatalogPO oldCatalogPO, CatalogEntity newCatalog, Long metalakeId) { + CatalogPO newCatalogPO = toCatalogPO(newCatalog, metalakeId); + Long lastVersion = oldCatalogPO.getLastVersion(); + // Will set the version to the last version + 1 when having some fields need be multiple version + Long nextVersion = lastVersion; + return new CatalogPO.Builder() + .withCatalogId(newCatalogPO.getCatalogId()) + .withCatalogName(newCatalogPO.getCatalogName()) + .withMetalakeId(metalakeId) + .withType(newCatalogPO.getType()) + .withProvider(newCatalogPO.getProvider()) + .withCatalogComment(newCatalogPO.getCatalogComment()) + .withProperties(newCatalogPO.getProperties()) + .withAuditInfo(newCatalogPO.getAuditInfo()) + .withCurrentVersion(nextVersion) + .withLastVersion(nextVersion) + .withDeletedAt(0L) + .build(); + } + /** * Convert {@link CatalogPO} to {@link CatalogEntity} * @@ -165,15 +218,16 @@ public static CatalogPO toCatalogPO(CatalogEntity catalogEntity, Long metalakeId public static CatalogEntity fromCatalogPO(CatalogPO catalogPO, Namespace namespace) { try { return CatalogEntity.builder() - .withId(catalogPO.getId()) + .withId(catalogPO.getCatalogId()) .withName(catalogPO.getCatalogName()) .withNamespace(namespace) .withType(Catalog.Type.valueOf(catalogPO.getType())) .withProvider(catalogPO.getProvider()) .withComment(catalogPO.getCatalogComment()) - .withProperties(JsonUtils.objectMapper().readValue(catalogPO.getProperties(), Map.class)) + .withProperties( + JsonUtils.anyFieldMapper().readValue(catalogPO.getProperties(), Map.class)) .withAuditInfo( - JsonUtils.objectMapper().readValue(catalogPO.getAuditInfo(), AuditInfo.class)) + JsonUtils.anyFieldMapper().readValue(catalogPO.getAuditInfo(), AuditInfo.class)) .build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to deserialize json object:", e); diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql index d4d31b0994b..da4c0fd41b8 100644 --- a/core/src/main/resources/mysql/mysql_init.sql +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -17,9 +17,8 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( UNIQUE KEY `uk_mn_del` (`metalake_name`, `deleted_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; -CREATE TABLE IF NOT EXISTS `catalog_meta` -( - `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', +CREATE TABLE IF NOT EXISTS `catalog_meta` ( + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', @@ -27,6 +26,9 @@ CREATE TABLE IF NOT EXISTS `catalog_meta` `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_cn_mid` (`catalog_name`, `metalake_id`) + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'catalog deleted at', + PRIMARY KEY (`catalog_id`), + UNIQUE KEY `uk_mid_cn_del` (`metalake_id`, `catalog_name`, `deleted_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'catalog metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index e1703c360ff..74271d0914e 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -14,10 +14,12 @@ import static com.datastrato.gravitino.Configs.ENTITY_STORE; import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; @@ -25,8 +27,10 @@ import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.exceptions.NonEmptyEntityException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import java.io.BufferedReader; @@ -130,6 +134,7 @@ public static void tearDown() { @Test public void testPutAndGet() throws IOException { + // metalake BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); entityStore.put(metalake, false); BaseMetalake insertedMetalake = @@ -153,10 +158,44 @@ public void testPutAndGet() throws IOException { entityStore.list(Namespace.empty(), BaseMetalake.class, Entity.EntityType.METALAKE).size()); assertEquals("test_metalake2", insertedMetalake1.name()); assertEquals("this is test2", insertedMetalake1.comment()); + + // catalog + CatalogEntity catalog = + createCatalog( + 1L, "test_catalog", Namespace.ofCatalog("test_metalake2"), "this is catalog test"); + entityStore.put(catalog, false); + CatalogEntity insertedCatalog = + entityStore.get(catalog.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class); + assertNotNull(insertedCatalog); + assertTrue(checkCatalogEquals(catalog, insertedCatalog)); + + // overwrite false + CatalogEntity duplicateCatalog = + createCatalog( + 1L, "test_catalog", Namespace.ofCatalog("test_metalake2"), "this is catalog test"); + assertThrows( + EntityAlreadyExistsException.class, () -> entityStore.put(duplicateCatalog, false)); + + // overwrite true + CatalogEntity overittenCatalog = + createCatalog( + 1L, "test_catalog1", Namespace.ofCatalog("test_metalake2"), "this is catalog test1"); + entityStore.put(overittenCatalog, true); + CatalogEntity insertedCatalog1 = + entityStore.get( + overittenCatalog.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class); + assertEquals( + 1, + entityStore + .list(overittenCatalog.namespace(), CatalogEntity.class, Entity.EntityType.CATALOG) + .size()); + assertEquals("test_catalog1", insertedCatalog1.name()); + assertEquals("this is catalog test1", insertedCatalog1.getComment()); } @Test public void testPutAndList() throws IOException { + // metalake BaseMetalake metalake1 = createMetalake(1L, "test_metalake1", "this is test 1"); BaseMetalake metalake2 = createMetalake(2L, "test_metalake2", "this is test 2"); List beforePutList = @@ -172,22 +211,101 @@ public void testPutAndList() throws IOException { assertEquals(2, metalakes.size()); assertTrue(checkMetalakeEquals(metalake1, metalakes.get(0))); assertTrue(checkMetalakeEquals(metalake2, metalakes.get(1))); + + // catalog + CatalogEntity catalog1 = + createCatalog( + 1L, "test_catalog1", Namespace.ofCatalog(metalake1.name()), "this is catalog 1"); + CatalogEntity catalog2 = + createCatalog( + 2L, "test_catalog2", Namespace.ofCatalog(metalake1.name()), "this is catalog 2"); + List beforeCatalogList = + entityStore.list(catalog1.namespace(), CatalogEntity.class, Entity.EntityType.CATALOG); + assertNotNull(beforeCatalogList); + assertEquals(0, beforeCatalogList.size()); + + entityStore.put(catalog1, false); + entityStore.put(catalog2, false); + List catalogEntities = + entityStore.list(catalog1.namespace(), CatalogEntity.class, Entity.EntityType.CATALOG); + assertNotNull(catalogEntities); + assertEquals(2, catalogEntities.size()); + assertTrue(checkCatalogEquals(catalog1, catalogEntities.get(0))); + assertTrue(checkCatalogEquals(catalog2, catalogEntities.get(1))); } @Test - public void testPutAndDelete() throws IOException { + public void testPutAndDelete() throws IOException, InterruptedException { + // metalake BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); entityStore.put(metalake, false); + assertNotNull( + entityStore.get(metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class)); entityStore.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE, false); assertThrows( NoSuchEntityException.class, () -> entityStore.get( metalake.nameIdentifier(), Entity.EntityType.METALAKE, BaseMetalake.class)); + + // sleep 1s to make delete_at seconds differently + Thread.sleep(1000); + + // test cascade delete + BaseMetalake metalake1 = createMetalake(2L, "test_metalake", "this is test"); + entityStore.put(metalake1, false); + CatalogEntity subCatalog = + createCatalog( + 1L, "test_catalog", Namespace.ofCatalog(metalake1.name()), "test cascade deleted"); + entityStore.put(subCatalog, false); + + // cascade is false + assertThrows( + NonEmptyEntityException.class, + () -> entityStore.delete(metalake1.nameIdentifier(), Entity.EntityType.METALAKE, false)); + + // cascade is true + entityStore.delete(metalake1.nameIdentifier(), Entity.EntityType.METALAKE, true); + assertFalse(entityStore.exists(metalake1.nameIdentifier(), Entity.EntityType.METALAKE)); + assertFalse(entityStore.exists(subCatalog.nameIdentifier(), Entity.EntityType.CATALOG)); + + // catalog + BaseMetalake metalake2 = createMetalake(3L, "test_metalake", "this is test"); + entityStore.put(metalake2, false); + CatalogEntity catalog = + createCatalog(2L, "test_catalog", Namespace.ofCatalog("test_metalake"), "this is test"); + entityStore.put(catalog, false); + assertNotNull( + entityStore.get(catalog.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class)); + entityStore.delete(catalog.nameIdentifier(), Entity.EntityType.CATALOG, false); + assertThrows( + NoSuchEntityException.class, + () -> + entityStore.get( + catalog.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class)); + + // sleep 1s to make delete_at seconds differently + Thread.sleep(1000); + + // test cascade delete + CatalogEntity catalog1 = + createCatalog( + 3L, "test_catalog1", Namespace.ofCatalog(metalake2.name()), "test cascade deleted"); + entityStore.put(catalog1, false); + + // cascade is false + assertThrows( + NonEmptyEntityException.class, + () -> entityStore.delete(metalake2.nameIdentifier(), Entity.EntityType.METALAKE, false)); + + // cascade is true + entityStore.delete(catalog1.nameIdentifier(), Entity.EntityType.CATALOG, true); + assertFalse(entityStore.exists(catalog1.nameIdentifier(), Entity.EntityType.CATALOG)); } @Test public void testPutAndUpdate() throws IOException { + // metalake BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); entityStore.put(metalake, false); @@ -236,6 +354,73 @@ public void testPutAndUpdate() throws IOException { assertEquals("test_metalake2", updatedMetalake.name()); assertEquals("this is test 2", updatedMetalake.comment()); assertEquals(changedAuditInfo.creator(), updatedMetalake.auditInfo().creator()); + + // catalog + CatalogEntity catalog = + createCatalog( + 1L, "test_catalog", Namespace.ofCatalog("test_metalake2"), "this is catalog test"); + entityStore.put(catalog, false); + assertThrows( + RuntimeException.class, + () -> + entityStore.update( + catalog.nameIdentifier(), + CatalogEntity.class, + Entity.EntityType.CATALOG, + c -> { + CatalogEntity.Builder builder = + CatalogEntity.builder() + // Change the id, which is not allowed + .withId(2L) + .withName("test_catalog2") + .withNamespace(Namespace.ofCatalog(updatedMetalake.name())) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment("this is catalog test 2") + .withProperties(new HashMap<>()) + .withAuditInfo((AuditInfo) c.auditInfo()); + return builder.build(); + })); + + CatalogEntity updatedCatalog = + entityStore.update( + catalog.nameIdentifier(), + CatalogEntity.class, + Entity.EntityType.CATALOG, + c -> { + CatalogEntity.Builder builder = + CatalogEntity.builder() + .withId(c.id()) + .withName("test_catalog2") + .withNamespace(Namespace.ofCatalog(updatedMetalake.name())) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment("this is catalog test 2") + .withProperties(new HashMap<>()) + .withAuditInfo(changedAuditInfo); + return builder.build(); + }); + CatalogEntity storedCatalog = + entityStore.get( + updatedCatalog.nameIdentifier(), Entity.EntityType.CATALOG, CatalogEntity.class); + assertEquals(catalog.id(), storedCatalog.id()); + assertEquals("test_catalog2", updatedCatalog.name()); + assertEquals("this is catalog test 2", updatedCatalog.getComment()); + assertEquals(changedAuditInfo.creator(), updatedCatalog.auditInfo().creator()); + } + + @Test + public void testPutAndExists() throws IOException, InterruptedException { + // metalake + BaseMetalake metalake = createMetalake(1L, "test_metalake", "this is test"); + entityStore.put(metalake, false); + assertTrue(entityStore.exists(metalake.nameIdentifier(), Entity.EntityType.METALAKE)); + + // catalog + CatalogEntity catalog = + createCatalog(1L, "test_catalog", Namespace.ofCatalog("test_metalake"), "this is test"); + entityStore.put(catalog, false); + assertTrue(entityStore.exists(catalog.nameIdentifier(), Entity.EntityType.CATALOG)); } private static BaseMetalake createMetalake(Long id, String name, String comment) { @@ -260,6 +445,33 @@ private static boolean checkMetalakeEquals(BaseMetalake expected, BaseMetalake a && expected.getVersion().equals(actual.getVersion()); } + private static CatalogEntity createCatalog( + Long id, String name, Namespace namespace, String comment) { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + return CatalogEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment(comment) + .withProperties(new HashMap<>()) + .withAuditInfo(auditInfo) + .build(); + } + + private static boolean checkCatalogEquals(CatalogEntity expected, CatalogEntity actual) { + return expected.id().equals(actual.id()) + && expected.name().equals(actual.name()) + && expected.namespace().equals(actual.namespace()) + && expected.getType().equals(actual.getType()) + && expected.getProvider().equals(actual.getProvider()) + && expected.getProperties() != null + && expected.getProperties().equals(actual.getProperties()) + && expected.auditInfo().equals(actual.auditInfo()); + } + private static void truncateAllTables() { try (SqlSession sqlSession = SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true)) { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java index 285e756c578..378f39918af 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java @@ -7,10 +7,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.storage.relational.po.CatalogPO; import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; import java.time.Instant; @@ -46,6 +50,28 @@ public void testFromMetalakePO() throws JsonProcessingException { assertEquals(expectedMetalake.getVersion(), convertedMetalake.getVersion()); } + @Test + public void testFromCatalogPO() throws JsonProcessingException { + CatalogPO catalogPO = createCatalogPO(1L, "test", 1L, "this is test"); + + CatalogEntity expectedCatalog = + createCatalog(1L, "test", Namespace.ofCatalog("test_metalake"), "this is test"); + + CatalogEntity convertedCatalog = + POConverters.fromCatalogPO(catalogPO, Namespace.ofCatalog("test_metalake")); + + // Assert + assertEquals(expectedCatalog.id(), convertedCatalog.id()); + assertEquals(expectedCatalog.name(), convertedCatalog.name()); + assertEquals(expectedCatalog.getComment(), convertedCatalog.getComment()); + assertEquals(expectedCatalog.getType(), convertedCatalog.getType()); + assertEquals(expectedCatalog.getProvider(), convertedCatalog.getProvider()); + assertEquals(expectedCatalog.namespace(), convertedCatalog.namespace()); + assertEquals( + expectedCatalog.getProperties().get("key"), convertedCatalog.getProperties().get("key")); + assertEquals(expectedCatalog.auditInfo().creator(), convertedCatalog.auditInfo().creator()); + } + @Test public void testFromMetalakePOs() throws JsonProcessingException { MetalakePO metalakePO1 = createMetalakePO(1L, "test", "this is test"); @@ -74,16 +100,59 @@ public void testFromMetalakePOs() throws JsonProcessingException { } @Test - public void testInitMetalakePOVersion() throws JsonProcessingException { - BaseMetalake metalakePO = createMetalake(1L, "test", "this is test"); - MetalakePO initPO = POConverters.initializeMetalakePOWithVersion(metalakePO); + public void testFromCatalogPOs() throws JsonProcessingException { + CatalogPO catalogPO1 = createCatalogPO(1L, "test", 1L, "this is test"); + CatalogPO catalogPO2 = createCatalogPO(2L, "test2", 1L, "this is test2"); + List catalogPOs = new ArrayList<>(Arrays.asList(catalogPO1, catalogPO2)); + List convertedCatalogs = + POConverters.fromCatalogPOs(catalogPOs, Namespace.ofCatalog("test_metalake")); + + CatalogEntity expectedCatalog1 = + createCatalog(1L, "test", Namespace.ofCatalog("test_metalake"), "this is test"); + CatalogEntity expectedCatalog2 = + createCatalog(2L, "test2", Namespace.ofCatalog("test_metalake"), "this is test2"); + List expectedCatalogs = + new ArrayList<>(Arrays.asList(expectedCatalog1, expectedCatalog2)); + + // Assert + int index = 0; + for (CatalogEntity catalog : convertedCatalogs) { + assertEquals(expectedCatalogs.get(index).id(), catalog.id()); + assertEquals(expectedCatalogs.get(index).name(), catalog.name()); + assertEquals(expectedCatalogs.get(index).getComment(), catalog.getComment()); + assertEquals(expectedCatalogs.get(index).getType(), catalog.getType()); + assertEquals(expectedCatalogs.get(index).getProvider(), catalog.getProvider()); + assertEquals(expectedCatalogs.get(index).namespace(), catalog.namespace()); + assertEquals( + expectedCatalogs.get(index).getProperties().get("key"), + catalog.getProperties().get("key")); + assertEquals( + expectedCatalogs.get(index).auditInfo().creator(), catalog.auditInfo().creator()); + index++; + } + } + + @Test + public void testInitMetalakePOVersion() { + BaseMetalake metalake = createMetalake(1L, "test", "this is test"); + MetalakePO initPO = POConverters.initializeMetalakePOWithVersion(metalake); assertEquals(1, initPO.getCurrentVersion()); assertEquals(1, initPO.getLastVersion()); assertEquals(0, initPO.getDeletedAt()); } @Test - public void testUpdateMetalakePOVersion() throws JsonProcessingException { + public void testInitCatalogPOVersion() { + CatalogEntity catalog = + createCatalog(1L, "test", Namespace.ofCatalog("test_metalake"), "this is test"); + CatalogPO initPO = POConverters.initializeCatalogPOWithVersion(catalog, 1L); + assertEquals(1, initPO.getCurrentVersion()); + assertEquals(1, initPO.getLastVersion()); + assertEquals(0, initPO.getDeletedAt()); + } + + @Test + public void testUpdateMetalakePOVersion() { BaseMetalake metalake = createMetalake(1L, "test", "this is test"); BaseMetalake updatedMetalake = createMetalake(1L, "test", "this is test2"); MetalakePO initPO = POConverters.initializeMetalakePOWithVersion(metalake); @@ -94,6 +163,20 @@ public void testUpdateMetalakePOVersion() throws JsonProcessingException { assertEquals("this is test2", updatePO.getMetalakeComment()); } + @Test + public void testUpdateCatalogPOVersion() { + CatalogEntity catalog = + createCatalog(1L, "test", Namespace.ofCatalog("test_metalake"), "this is test"); + CatalogEntity updatedCatalog = + createCatalog(1L, "test", Namespace.ofCatalog("test_metalake"), "this is test2"); + CatalogPO initPO = POConverters.initializeCatalogPOWithVersion(catalog, 1L); + CatalogPO updatePO = POConverters.updateCatalogPOWithVersion(initPO, updatedCatalog, 1L); + assertEquals(1, initPO.getCurrentVersion()); + assertEquals(1, initPO.getLastVersion()); + assertEquals(0, initPO.getDeletedAt()); + assertEquals("this is test2", updatePO.getCatalogComment()); + } + @Test public void testToMetalakePO() throws JsonProcessingException { BaseMetalake metalake = createMetalake(1L, "test", "this is test"); @@ -111,6 +194,26 @@ public void testToMetalakePO() throws JsonProcessingException { assertEquals(expectedMetalakePO.getSchemaVersion(), actualMetalakePO.getSchemaVersion()); } + @Test + public void testToCatalogPO() throws JsonProcessingException { + CatalogEntity catalog = + createCatalog(1L, "test", Namespace.ofCatalog("test_metalake"), "this is test"); + + CatalogPO expectedCatalogPO = createCatalogPO(1L, "test", 1L, "this is test"); + + CatalogPO actualCatalogPO = POConverters.toCatalogPO(catalog, 1L); + + // Assert + assertEquals(expectedCatalogPO.getCatalogId(), actualCatalogPO.getCatalogId()); + assertEquals(expectedCatalogPO.getMetalakeId(), actualCatalogPO.getMetalakeId()); + assertEquals(expectedCatalogPO.getCatalogName(), actualCatalogPO.getCatalogName()); + assertEquals(expectedCatalogPO.getType(), actualCatalogPO.getType()); + assertEquals(expectedCatalogPO.getProvider(), actualCatalogPO.getProvider()); + assertEquals(expectedCatalogPO.getCatalogComment(), actualCatalogPO.getCatalogComment()); + assertEquals(expectedCatalogPO.getProperties(), actualCatalogPO.getProperties()); + assertEquals(expectedCatalogPO.getAuditInfo(), actualCatalogPO.getAuditInfo()); + } + private static BaseMetalake createMetalake(Long id, String name, String comment) { AuditInfo auditInfo = AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); @@ -141,4 +244,40 @@ private static MetalakePO createMetalakePO(Long id, String name, String comment) .withSchemaVersion(JsonUtils.anyFieldMapper().writeValueAsString(SchemaVersion.V_0_1)) .build(); } + + private static CatalogEntity createCatalog( + Long id, String name, Namespace namespace, String comment) { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); + Map properties = new HashMap<>(); + properties.put("key", "value"); + return CatalogEntity.builder() + .withId(id) + .withName(name) + .withNamespace(namespace) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment(comment) + .withProperties(properties) + .withAuditInfo(auditInfo) + .build(); + } + + private static CatalogPO createCatalogPO(Long id, String name, Long metalakeId, String comment) + throws JsonProcessingException { + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); + Map properties = new HashMap<>(); + properties.put("key", "value"); + return new CatalogPO.Builder() + .withCatalogId(id) + .withCatalogName(name) + .withMetalakeId(metalakeId) + .withType(Catalog.Type.RELATIONAL.name()) + .withProvider("test") + .withCatalogComment(comment) + .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(properties)) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(auditInfo)) + .build(); + } } diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index 6107ac7e105..f0421bf7511 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( CREATE TABLE IF NOT EXISTS `catalog_meta` ( - `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', @@ -28,6 +28,9 @@ CREATE TABLE IF NOT EXISTS `catalog_meta` `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', - PRIMARY KEY (id), - CONSTRAINT uk_cn_mid UNIQUE (catalog_name, metalake_id) + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'catalog deleted at', + PRIMARY KEY (catalog_id), + CONSTRAINT uk_mid_cn_del UNIQUE (metalake_id, catalog_name, deleted_at) ) ENGINE=InnoDB; \ No newline at end of file From 70a91d08c8f0629db2ab2d47043118c3c071bb41 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 27 Feb 2024 11:59:59 +0800 Subject: [PATCH 32/34] fix comments --- .../exceptions/NoSuchEntityException.java | 4 +- .../service/CatalogMetaService.java | 37 +++++++------------ .../service/MetalakeMetaService.java | 11 ++++-- core/src/main/resources/mysql/mysql_init.sql | 4 +- .../relational/TestRelationalEntityStore.java | 12 +++++- core/src/test/resources/h2/h2-init.sql | 4 +- 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java index e3f391162fc..27ba0569e3b 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java @@ -9,8 +9,8 @@ /** This exception is thrown when an entity is not found. */ public class NoSuchEntityException extends RuntimeException { - /** The no such an entity message for the exception. */ - public static final String NO_SUCH_AN_ENTITY_MESSAGE = "No such an entity: %s"; + /** The no such entity message for the exception. */ + public static final String NO_SUCH_ENTITY_MESSAGE = "No such entity: %s"; /** * Constructs a new NoSuchEntityException. diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java index 2a9f9236e54..2d329c700d1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -36,7 +36,7 @@ public static CatalogMetaService getInstance() { private CatalogMetaService() {} public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) { - checkCatalogNamespaceWithIdentifier(identifier); + NameIdentifier.checkCatalog(identifier); String metalakeName = identifier.namespace().level(0); String catalogName = identifier.name(); Long metalakeId = @@ -44,7 +44,7 @@ public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) { MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.namespace().toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, identifier.namespace().toString()); } CatalogPO catalogPO = SessionUtils.getWithoutCommit( @@ -52,20 +52,20 @@ public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) { mapper -> mapper.selectCatalogMetaByMetalakeIdAndName(metalakeId, catalogName)); if (catalogPO == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, identifier.toString()); } return POConverters.fromCatalogPO(catalogPO, identifier.namespace()); } public List listCatalogsByNamespace(Namespace namespace) { - checkCatalogNamespace(namespace); + Namespace.checkCatalog(namespace); String metalakeName = namespace.level(0); Long metalakeId = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, namespace.toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, namespace.toString()); } List catalogPOS = SessionUtils.getWithoutCommit( @@ -75,14 +75,14 @@ public List listCatalogsByNamespace(Namespace namespace) { public void insertCatalog(CatalogEntity catalogEntity, boolean overwrite) { try { - checkCatalogNamespaceWithIdentifier(catalogEntity.nameIdentifier()); + NameIdentifier.checkCatalog(catalogEntity.nameIdentifier()); Long metalakeId = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(catalogEntity.namespace().level(0))); if (metalakeId == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, catalogEntity.namespace().toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, catalogEntity.namespace().toString()); } SessionUtils.doWithCommit( CatalogMetaMapper.class, @@ -103,8 +103,7 @@ public void insertCatalog(CatalogEntity catalogEntity, boolean overwrite) { // SQL violates the constraints of `primary key` and `unique key`. // We simply think that the entity already exists at this time. throw new EntityAlreadyExistsException( - String.format( - "Catalog entity: %s already exists", catalogEntity.nameIdentifier().name())); + String.format("Catalog entity: %s already exists", catalogEntity.nameIdentifier())); } throw re; } @@ -112,7 +111,7 @@ public void insertCatalog(CatalogEntity catalogEntity, boolean overwrite) { public CatalogEntity updateCatalog( NameIdentifier identifier, Function updater) throws IOException { - checkCatalogNamespaceWithIdentifier(identifier); + NameIdentifier.checkCatalog(identifier); String metalakeName = identifier.namespace().level(0); String catalogName = identifier.name(); Long metalakeId = @@ -120,7 +119,7 @@ public CatalogEntity updateCatalog( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.namespace().toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, identifier.namespace().toString()); } CatalogPO oldCatalogPO = @@ -129,7 +128,7 @@ public CatalogEntity updateCatalog( mapper -> mapper.selectCatalogMetaByMetalakeIdAndName(metalakeId, catalogName)); if (oldCatalogPO == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, identifier.toString()); } CatalogEntity oldCatalogEntity = @@ -157,7 +156,7 @@ public CatalogEntity updateCatalog( } public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { - checkCatalogNamespaceWithIdentifier(identifier); + NameIdentifier.checkCatalog(identifier); String metalakeName = identifier.namespace().level(0); String catalogName = identifier.name(); Long metalakeId = @@ -165,7 +164,7 @@ public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.namespace().toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, identifier.namespace().toString()); } Long catalogId = SessionUtils.getWithoutCommit( @@ -190,14 +189,4 @@ public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { } return true; } - - private void checkCatalogNamespaceWithIdentifier(NameIdentifier identifier) { - Preconditions.checkArgument( - identifier.hasNamespace() && identifier.namespace().levels().length == 1, - "Only support one level namespace"); - } - - private void checkCatalogNamespace(Namespace namespace) { - Preconditions.checkArgument(namespace.levels().length == 1, "Only support one level namespace"); - } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 0ac0dfe7f60..46810f39514 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -46,18 +46,20 @@ public List listMetalakes() { } public BaseMetalake getMetalakeByIdentifier(NameIdentifier ident) { + NameIdentifier.checkMetalake(ident); MetalakePO metalakePO = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); if (metalakePO == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, ident.toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, ident.toString()); } return POConverters.fromMetalakePO(metalakePO); } public void insertMetalake(BaseMetalake baseMetalake, boolean overwrite) { try { + NameIdentifier.checkMetalake(baseMetalake.nameIdentifier()); SessionUtils.doWithCommit( MetalakeMetaMapper.class, mapper -> { @@ -77,8 +79,7 @@ public void insertMetalake(BaseMetalake baseMetalake, boolean overwrite) { // SQL violates the constraints of `primary key` and `unique key`. // We simply think that the entity already exists at this time. throw new EntityAlreadyExistsException( - String.format( - "Metalake entity: %s already exists", baseMetalake.nameIdentifier().name())); + String.format("Metalake entity: %s already exists", baseMetalake.nameIdentifier())); } throw re; } @@ -86,12 +87,13 @@ public void insertMetalake(BaseMetalake baseMetalake, boolean overwrite) { public BaseMetalake updateMetalake( NameIdentifier ident, Function updater) throws IOException { + NameIdentifier.checkMetalake(ident); MetalakePO oldMetalakePO = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); if (oldMetalakePO == null) { throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, ident.toString()); + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, ident.toString()); } BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); @@ -116,6 +118,7 @@ public BaseMetalake updateMetalake( } public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { + NameIdentifier.checkMetalake(ident); Long metalakeId = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(ident.name())); diff --git a/core/src/main/resources/mysql/mysql_init.sql b/core/src/main/resources/mysql/mysql_init.sql index da4c0fd41b8..613d0b692b3 100644 --- a/core/src/main/resources/mysql/mysql_init.sql +++ b/core/src/main/resources/mysql/mysql_init.sql @@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'metalake deleted at', PRIMARY KEY (`metalake_id`), UNIQUE KEY `uk_mn_del` (`metalake_name`, `deleted_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS `catalog_meta` ( `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'catalog deleted at', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'catalog deleted at', PRIMARY KEY (`catalog_id`), UNIQUE KEY `uk_mid_cn_del` (`metalake_id`, `catalog_name`, `deleted_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'catalog metadata'; \ No newline at end of file diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index 74271d0914e..5ad7118f7c0 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -47,9 +47,11 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -206,7 +208,10 @@ public void testPutAndList() throws IOException { entityStore.put(metalake1, false); entityStore.put(metalake2, false); List metalakes = - entityStore.list(metalake1.namespace(), BaseMetalake.class, Entity.EntityType.METALAKE); + entityStore.list(metalake1.namespace(), BaseMetalake.class, Entity.EntityType.METALAKE) + .stream() + .sorted(Comparator.comparing(BaseMetalake::id)) + .collect(Collectors.toList()); assertNotNull(metalakes); assertEquals(2, metalakes.size()); assertTrue(checkMetalakeEquals(metalake1, metalakes.get(0))); @@ -227,7 +232,10 @@ public void testPutAndList() throws IOException { entityStore.put(catalog1, false); entityStore.put(catalog2, false); List catalogEntities = - entityStore.list(catalog1.namespace(), CatalogEntity.class, Entity.EntityType.CATALOG); + entityStore.list(catalog1.namespace(), CatalogEntity.class, Entity.EntityType.CATALOG) + .stream() + .sorted(Comparator.comparing(CatalogEntity::id)) + .collect(Collectors.toList()); assertNotNull(catalogEntities); assertEquals(2, catalogEntities.size()); assertTrue(checkCatalogEquals(catalog1, catalogEntities.get(0))); diff --git a/core/src/test/resources/h2/h2-init.sql b/core/src/test/resources/h2/h2-init.sql index f0421bf7511..9f47ee11dfe 100644 --- a/core/src/test/resources/h2/h2-init.sql +++ b/core/src/test/resources/h2/h2-init.sql @@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS `metalake_meta` ( `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'metalake deleted at', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'metalake deleted at', PRIMARY KEY (metalake_id), CONSTRAINT uk_mn_del UNIQUE (metalake_name, deleted_at) ) ENGINE = InnoDB; @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS `catalog_meta` `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 NULL COMMENT 'catalog deleted at', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'catalog deleted at', PRIMARY KEY (catalog_id), CONSTRAINT uk_mid_cn_del UNIQUE (metalake_id, catalog_name, deleted_at) ) ENGINE=InnoDB; \ No newline at end of file From 25a60b2b26d469a9ed1be23461c14c4dbc82336d Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 27 Feb 2024 19:00:25 +0800 Subject: [PATCH 33/34] catch constraint exception when update --- .../service/CatalogMetaService.java | 30 ++++++++++++++----- .../service/MetalakeMetaService.java | 23 +++++++++++--- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java index 2d329c700d1..27e22d47f9a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -139,14 +139,28 @@ public CatalogEntity updateCatalog( "The updated catalog entity id: %s should be same with the catalog entity id before: %s", newEntity.id(), oldCatalogEntity.id()); - - Integer updateResult = - SessionUtils.doWithCommitAndFetchResult( - CatalogMetaMapper.class, - mapper -> - mapper.updateCatalogMeta( - POConverters.updateCatalogPOWithVersion(oldCatalogPO, newEntity, metalakeId), - oldCatalogPO)); + Integer updateResult; + try { + updateResult = + SessionUtils.doWithCommitAndFetchResult( + CatalogMetaMapper.class, + mapper -> + mapper.updateCatalogMeta( + POConverters.updateCatalogPOWithVersion(oldCatalogPO, newEntity, metalakeId), + oldCatalogPO)); + } catch (RuntimeException re) { + if (re.getCause() != null + && re.getCause().getCause() != null + && re.getCause().getCause() instanceof SQLIntegrityConstraintViolationException) { + // TODO We should make more fine-grained exception judgments + // Usually throwing `SQLIntegrityConstraintViolationException` means that + // SQL violates the constraints of `primary key` and `unique key`. + // We simply think that the entity already exists at this time. + throw new EntityAlreadyExistsException( + String.format("Catalog entity: %s already exists", newEntity.nameIdentifier())); + } + throw re; + } if (updateResult > 0) { return newEntity; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 46810f39514..3850e1e47e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -105,11 +105,26 @@ public BaseMetalake updateMetalake( oldMetalakeEntity.id()); MetalakePO newMetalakePO = POConverters.updateMetalakePOWithVersion(oldMetalakePO, newMetalakeEntity); + Integer updateResult; + try { + updateResult = + SessionUtils.doWithCommitAndFetchResult( + MetalakeMetaMapper.class, + mapper -> mapper.updateMetalakeMeta(newMetalakePO, oldMetalakePO)); + } catch (RuntimeException re) { + if (re.getCause() != null + && re.getCause().getCause() != null + && re.getCause().getCause() instanceof SQLIntegrityConstraintViolationException) { + // TODO We should make more fine-grained exception judgments + // Usually throwing `SQLIntegrityConstraintViolationException` means that + // SQL violates the constraints of `primary key` and `unique key`. + // We simply think that the entity already exists at this time. + throw new EntityAlreadyExistsException( + String.format("Catalog entity: %s already exists", newMetalakeEntity.nameIdentifier())); + } + throw re; + } - Integer updateResult = - SessionUtils.doWithCommitAndFetchResult( - MetalakeMetaMapper.class, - mapper -> mapper.updateMetalakeMeta(newMetalakePO, oldMetalakePO)); if (updateResult > 0) { return newMetalakeEntity; } else { From 4adcac875f847bb5fc07a6f0f9b146db140938c8 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 27 Feb 2024 20:42:02 +0800 Subject: [PATCH 34/34] add ut for update unique key constraints --- .../relational/TestRelationalEntityStore.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java index 5ad7118f7c0..ccf84585d20 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestRelationalEntityStore.java @@ -363,6 +363,27 @@ public void testPutAndUpdate() throws IOException { assertEquals("this is test 2", updatedMetalake.comment()); assertEquals(changedAuditInfo.creator(), updatedMetalake.auditInfo().creator()); + BaseMetalake metalake3 = createMetalake(3L, "test_metalake3", "this is test 3"); + entityStore.put(metalake3, false); + assertThrows( + EntityAlreadyExistsException.class, + () -> + entityStore.update( + metalake3.nameIdentifier(), + BaseMetalake.class, + Entity.EntityType.METALAKE, + m -> { + BaseMetalake.Builder builder = + new BaseMetalake.Builder() + .withId(metalake3.id()) + .withName("test_metalake2") + .withComment(metalake3.comment()) + .withProperties(new HashMap<>()) + .withAuditInfo((AuditInfo) m.auditInfo()) + .withVersion(m.getVersion()); + return builder.build(); + })); + // catalog CatalogEntity catalog = createCatalog( @@ -415,6 +436,31 @@ public void testPutAndUpdate() throws IOException { assertEquals("test_catalog2", updatedCatalog.name()); assertEquals("this is catalog test 2", updatedCatalog.getComment()); assertEquals(changedAuditInfo.creator(), updatedCatalog.auditInfo().creator()); + + CatalogEntity catalog3 = + createCatalog( + 3L, "test_catalog3", Namespace.ofCatalog("test_metalake2"), "this is catalog test 3"); + entityStore.put(catalog3, false); + assertThrows( + EntityAlreadyExistsException.class, + () -> + entityStore.update( + catalog3.nameIdentifier(), + CatalogEntity.class, + Entity.EntityType.CATALOG, + c -> { + CatalogEntity.Builder builder = + CatalogEntity.builder() + .withId(catalog3.id()) + .withName("test_catalog2") + .withNamespace(Namespace.ofCatalog(updatedMetalake.name())) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment(catalog3.getComment()) + .withProperties(new HashMap<>()) + .withAuditInfo((AuditInfo) c.auditInfo()); + return builder.build(); + })); } @Test