Skip to content

Commit

Permalink
[#3924] improvemet(core): Switch default entity store from KV to rela…
Browse files Browse the repository at this point in the history
…tional (#3954)

### What changes were proposed in this pull request?

Change configurations about default entity store and change default
entity store from KV to relational.

### Why are the changes needed?

We are going to make KV entity deprecated. 

Fix: #3924 

### Does this PR introduce _any_ user-facing change?

N/A.

### How was this patch tested?

Existing test can cover it.
  • Loading branch information
yuqi1129 authored Jul 1, 2024
1 parent e5b3eb9 commit f39fa4a
Show file tree
Hide file tree
Showing 19 changed files with 101 additions and 68 deletions.
20 changes: 12 additions & 8 deletions conf/gravitino.conf.template
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,19 @@ gravitino.server.webserver.requestHeaderSize = 131072
gravitino.server.webserver.responseHeaderSize = 131072

# THE CONFIGURATION FOR Gravitino ENTITY STORE
# The entity store to use, kv or relational
gravitino.entity.store = kv
# The entity store to use, we only supports relational
gravitino.entity.store = relational
# The backend for the entity store, we only supports JDBC
gravitino.entity.store.relational = JDBCBackend

# The RocksDB entity store
gravitino.entity.store.kv = RocksDBKvBackend
# The storage path for RocksDB storage implementation, it supports both absolute and relative path,
# If the value is a relative path, the final path is "${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}", default value
# is "${GRAVITINO_HOME}/data/rocksdb", please uncomment and change it in your production environment.
# gravitino.entity.store.kv.rocksdbPath = data/rocksdb
# The JDBC URL for the entity store
gravitino.entity.store.relational.jdbcUrl = jdbc:h2
# The JDBC driver class name
gravitino.entity.store.relational.jdbcDriver = org.h2.Driver
# The JDBC user name
gravitino.entity.store.relational.jdbcUser = gravitino
# The JDBC password
gravitino.entity.store.relational.jdbcPassword = gravitino

# THE CONFIGURATION FOR Gravitino CATALOG
# The interval in milliseconds to evict the catalog cache
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/com/datastrato/gravitino/Configs.java
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ private Configs() {}
.doc("Which storage implementation to use")
.version(ConfigConstants.VERSION_0_1_0)
.stringConf()
.createWithDefault(DEFAULT_ENTITY_STORE);
.createWithDefault(RELATIONAL_ENTITY_STORE);

public static final ConfigEntry<String> ENTITY_KV_STORE =
new ConfigBuilder(ENTITY_KV_STORE_KEY)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,14 +149,16 @@ Integer updateCatalogMeta(
@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE catalog_id = #{catalogId} AND deleted_at = 0")
Integer softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
Integer softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,28 +229,32 @@ Integer updateFilesetMeta(
@Update(
"UPDATE "
+ META_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
Integer softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

@Update(
"UPDATE "
+ META_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE catalog_id = #{catalogId} AND deleted_at = 0")
Integer softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId);

@Update(
"UPDATE "
+ META_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0")
Integer softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId);

@Update(
"UPDATE "
+ META_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE fileset_id = #{filesetId} AND deleted_at = 0")
Integer softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,28 +77,32 @@ void insertFilesetVersionOnDuplicateKeyUpdate(
@Update(
"UPDATE "
+ VERSION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
Integer softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId);

@Update(
"UPDATE "
+ VERSION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE catalog_id = #{catalogId} AND deleted_at = 0")
Integer softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId);

@Update(
"UPDATE "
+ VERSION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0")
Integer softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId);

@Update(
"UPDATE "
+ VERSION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE fileset_id = #{filesetId} AND deleted_at = 0")
Integer softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId);

Expand All @@ -122,7 +126,8 @@ List<FilesetMaxVersionPO> selectFilesetVersionsByRetentionCount(
@Update(
"UPDATE "
+ VERSION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}")
Integer softDeleteFilesetVersionsByRetentionLine(
@Param("filesetId") Long filesetId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,16 @@ GroupPO selectGroupMetaByMetalakeIdAndName(
@Update(
"UPDATE "
+ GROUP_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE group_id = #{groupId} AND deleted_at = 0")
void softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId);

@Update(
"UPDATE "
+ GROUP_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
void softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,15 +75,17 @@ void batchInsertGroupRoleRelOnDuplicateKeyUpdate(
@Update(
"UPDATE "
+ GROUP_ROLE_RELATION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE group_id = #{groupId} AND deleted_at = 0")
void softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId);

@Update({
"<script>",
"UPDATE "
+ GROUP_ROLE_RELATION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE group_id = #{groupId} AND role_id in (",
"<foreach collection='roleIds' item='roleId' separator=','>",
"#{roleId}",
Expand All @@ -97,7 +99,8 @@ void softDeleteGroupRoleRelByGroupAndRoles(
@Update(
"UPDATE "
+ GROUP_ROLE_RELATION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE group_id IN (SELECT group_id FROM "
+ GROUP_TABLE_NAME
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)"
Expand All @@ -107,7 +110,8 @@ void softDeleteGroupRoleRelByGroupAndRoles(
@Update(
"UPDATE "
+ GROUP_ROLE_RELATION_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE role_id = #{roleId} AND deleted_at = 0")
void softDeleteGroupRoleRelByRoleId(Long roleId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,8 @@ Integer updateMetalakeMeta(
@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
Integer softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,14 +120,16 @@ Long selectRoleIdByMetalakeIdAndName(
@Update(
"UPDATE "
+ ROLE_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE role_id = #{roleId} AND deleted_at = 0")
void softDeleteRoleMetaByRoleId(Long roleId);

@Update(
"UPDATE "
+ ROLE_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,21 +144,24 @@ Integer updateSchemaMeta(
@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0")
Integer softDeleteSchemaMetasBySchemaId(@Param("schemaId") Long schemaId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
Integer softDeleteSchemaMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE catalog_id = #{catalogId} AND deleted_at = 0")
Integer softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,16 @@ void batchInsertSecurableObjects(
@Update(
"UPDATE "
+ SECURABLE_OBJECT_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE role_id = #{roleId} AND deleted_at = 0")
void softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId);

@Update(
"UPDATE "
+ SECURABLE_OBJECT_TABLE_NAME
+ " ob SET ob.deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " ob SET ob.deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " where exists ( select * from "
+ ROLE_TABLE_NAME
+ " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,28 +139,32 @@ Integer updateTableMeta(
@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE table_id = #{tableId} AND deleted_at = 0")
Integer softDeleteTableMetasByTableId(@Param("tableId") Long tableId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
Integer softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE catalog_id = #{catalogId} AND deleted_at = 0")
Integer softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0")
Integer softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,28 +140,32 @@ Long selectTopicIdBySchemaIdAndName(
@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE topic_id = #{topicId} AND deleted_at = 0")
Integer softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE catalog_id = #{catalogId} AND deleted_at = 0")
Integer softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
Integer softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

@Update(
"UPDATE "
+ TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0")
Integer softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,16 @@ UserPO selectUserMetaByMetalakeIdAndName(
@Update(
"UPDATE "
+ USER_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE user_id = #{userId} AND deleted_at = 0")
void softDeleteUserMetaByUserId(@Param("userId") Long userId);

@Update(
"UPDATE "
+ USER_TABLE_NAME
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+ " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
void softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

Expand Down
Loading

0 comments on commit f39fa4a

Please sign in to comment.