From e9929c875e48ec02dd001cc70b80b3de87ce5435 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 6 Sep 2024 00:05:34 +0800 Subject: [PATCH 1/8] [#4832] feat(core): Add basic framework to support multiple JDBC backend. (#4833) ### What changes were proposed in this pull request? 1. Add a framework to support multiple JDBC backend 2. Modify all the mapper classes related to metalake, catalog, schema, filesetset, table, topic and use new framework, others will use another PR to avoid a large PR. ### Why are the changes needed? To support more JDBC storage backend. Fix: #4832 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Existing tests. --- .../storage/relational/JDBCBackend.java | 20 +- .../mapper/CatalogMetaBaseSQLProvider.java | 173 +++++++++++++++ .../relational/mapper/CatalogMetaMapper.java | 151 +++---------- .../mapper/CatalogMetaSQLProviderFactory.java | 97 +++++++++ .../mapper/FilesetMetaBaseSQLProvider.java | 200 ++++++++++++++++++ .../relational/mapper/FilesetMetaMapper.java | 178 +++------------- .../mapper/FilesetMetaSQLProviderFactory.java | 104 +++++++++ .../mapper/FilesetVersionBaseSQLProvider.java | 135 ++++++++++++ .../mapper/FilesetVersionMapper.java | 118 +++-------- .../FilesetVersionSQLProviderFactory.java | 93 ++++++++ .../mapper/MetalakeMetaBaseSQLProvider.java | 152 +++++++++++++ .../relational/mapper/MetalakeMetaMapper.java | 127 ++--------- .../MetalakeMetaSQLProviderFactory.java | 91 ++++++++ .../mapper/SchemaMetaBaseSQLProvider.java | 174 +++++++++++++++ .../relational/mapper/SchemaMetaMapper.java | 155 +++----------- .../mapper/SchemaMetaSQLProviderFactory.java | 98 +++++++++ .../mapper/TableMetaBaseSQLProvider.java | 178 ++++++++++++++++ .../relational/mapper/TableMetaMapper.java | 159 +++----------- .../mapper/TableMetaSQLProviderFactory.java | 102 +++++++++ .../mapper/TopicMetaBaseSQLProvider.java | 189 +++++++++++++++++ .../relational/mapper/TopicMetaMapper.java | 169 +++------------ .../mapper/TopicMetaSQLProviderFactory.java | 103 +++++++++ .../session/SqlSessionFactoryHelper.java | 6 +- 23 files changed, 2113 insertions(+), 859 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 81e111a289b..b23c7667388 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -417,9 +417,10 @@ public void insertRelation( } } - enum JDBCBackendType { + public enum JDBCBackendType { H2(true), - MYSQL(false); + MYSQL(false), + POSTGRESQL(false); private final boolean embedded; @@ -432,10 +433,25 @@ public static JDBCBackendType fromURI(String jdbcURI) { return JDBCBackendType.H2; } else if (jdbcURI.startsWith("jdbc:mysql")) { return JDBCBackendType.MYSQL; + } else if (jdbcURI.startsWith("jdbc:postgresql")) { + return JDBCBackendType.POSTGRESQL; } else { throw new IllegalArgumentException("Unknown JDBC URI: " + jdbcURI); } } + + public static JDBCBackendType fromString(String jdbcType) { + switch (jdbcType) { + case "h2": + return JDBCBackendType.H2; + case "mysql": + return JDBCBackendType.MYSQL; + case "postgresql": + return JDBCBackendType.POSTGRESQL; + default: + throw new IllegalArgumentException("Unknown JDBC type: " + jdbcType); + } + } } /** Start JDBC database if necessary. For example, start the H2 database if the backend is H2. */ diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaBaseSQLProvider.java new file mode 100644 index 00000000000..cce5783898f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaBaseSQLProvider.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.CatalogPO; +import org.apache.ibatis.annotations.Param; + +public class CatalogMetaBaseSQLProvider { + public String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String selectCatalogIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return "SELECT catalog_id as catalogId FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0"; + } + + public String selectCatalogMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0"; + } + + public String selectCatalogMetaById(@Param("catalogId") Long catalogId) { + return "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO) { + return "INSERT INTO " + + TABLE_NAME + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{catalogMeta.catalogId}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + + " )"; + } + + public String insertCatalogMetaOnDuplicateKeyUpdate(@Param("catalogMeta") CatalogPO catalogPO) { + return "INSERT INTO " + + TABLE_NAME + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{catalogMeta.catalogId}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " catalog_name = #{catalogMeta.catalogName}," + + " metalake_id = #{catalogMeta.metalakeId}," + + " type = #{catalogMeta.type}," + + " provider = #{catalogMeta.provider}," + + " catalog_comment = #{catalogMeta.catalogComment}," + + " properties = #{catalogMeta.properties}," + + " audit_info = #{catalogMeta.auditInfo}," + + " current_version = #{catalogMeta.currentVersion}," + + " last_version = #{catalogMeta.lastVersion}," + + " deleted_at = #{catalogMeta.deletedAt}"; + } + + public String updateCatalogMeta( + @Param("newCatalogMeta") CatalogPO newCatalogPO, + @Param("oldCatalogMeta") CatalogPO oldCatalogPO) { + return "UPDATE " + + TABLE_NAME + + " SET catalog_name = #{newCatalogMeta.catalogName}," + + " metalake_id = #{newCatalogMeta.metalakeId}," + + " type = #{newCatalogMeta.type}," + + " provider = #{newCatalogMeta.provider}," + + " catalog_comment = #{newCatalogMeta.catalogComment}," + + " properties = #{newCatalogMeta.properties}," + + " audit_info = #{newCatalogMeta.auditInfo}," + + " current_version = #{newCatalogMeta.currentVersion}," + + " last_version = #{newCatalogMeta.lastVersion}," + + " deleted_at = #{newCatalogMeta.deletedAt}" + + " WHERE catalog_id = #{oldCatalogMeta.catalogId}" + + " AND catalog_name = #{oldCatalogMeta.catalogName}" + + " AND metalake_id = #{oldCatalogMeta.metalakeId}" + + " AND type = #{oldCatalogMeta.type}" + + " AND provider = #{oldCatalogMeta.provider}" + + " AND (catalog_comment = #{oldCatalogMeta.catalogComment} " + + " OR (catalog_comment IS NULL and #{oldCatalogMeta.catalogComment} IS NULL))" + + " AND properties = #{oldCatalogMeta.properties}" + + " AND audit_info = #{oldCatalogMeta.auditInfo}" + + " AND current_version = #{oldCatalogMeta.currentVersion}" + + " AND last_version = #{oldCatalogMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteCatalogMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java index faedbcd9642..01cadbb6e8b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.CatalogPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for catalog meta operation SQLs. @@ -38,149 +38,50 @@ public interface CatalogMetaMapper { String TABLE_NAME = "catalog_meta"; - @Select( - "SELECT catalog_id as catalogId, catalog_name as catalogName," - + " metalake_id as metalakeId, type, provider," - + " catalog_comment as catalogComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "listCatalogPOsByMetalakeId") List listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId); - @Select( - "SELECT catalog_id as catalogId FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0") + @SelectProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "selectCatalogIdByMetalakeIdAndName") Long selectCatalogIdByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); - @Select( - "SELECT catalog_id as catalogId, catalog_name as catalogName," - + " metalake_id as metalakeId, type, provider," - + " catalog_comment as catalogComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0") + @SelectProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "selectCatalogMetaByMetalakeIdAndName") CatalogPO selectCatalogMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); - @Select( - "SELECT catalog_id as catalogId, catalog_name as catalogName," - + " metalake_id as metalakeId, type, provider," - + " catalog_comment as catalogComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "selectCatalogMetaById") CatalogPO selectCatalogMetaById(@Param("catalogId") Long catalogId); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(catalog_id, catalog_name, metalake_id," - + " type, provider, catalog_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{catalogMeta.catalogId}," - + " #{catalogMeta.catalogName}," - + " #{catalogMeta.metalakeId}," - + " #{catalogMeta.type}," - + " #{catalogMeta.provider}," - + " #{catalogMeta.catalogComment}," - + " #{catalogMeta.properties}," - + " #{catalogMeta.auditInfo}," - + " #{catalogMeta.currentVersion}," - + " #{catalogMeta.lastVersion}," - + " #{catalogMeta.deletedAt}" - + " )") + @InsertProvider(type = CatalogMetaSQLProviderFactory.class, method = "insertCatalogMeta") void insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(catalog_id, catalog_name, metalake_id," - + " type, provider, catalog_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{catalogMeta.catalogId}," - + " #{catalogMeta.catalogName}," - + " #{catalogMeta.metalakeId}," - + " #{catalogMeta.type}," - + " #{catalogMeta.provider}," - + " #{catalogMeta.catalogComment}," - + " #{catalogMeta.properties}," - + " #{catalogMeta.auditInfo}," - + " #{catalogMeta.currentVersion}," - + " #{catalogMeta.lastVersion}," - + " #{catalogMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " catalog_name = #{catalogMeta.catalogName}," - + " metalake_id = #{catalogMeta.metalakeId}," - + " type = #{catalogMeta.type}," - + " provider = #{catalogMeta.provider}," - + " catalog_comment = #{catalogMeta.catalogComment}," - + " properties = #{catalogMeta.properties}," - + " audit_info = #{catalogMeta.auditInfo}," - + " current_version = #{catalogMeta.currentVersion}," - + " last_version = #{catalogMeta.lastVersion}," - + " deleted_at = #{catalogMeta.deletedAt}") + @InsertProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "insertCatalogMetaOnDuplicateKeyUpdate") void insertCatalogMetaOnDuplicateKeyUpdate(@Param("catalogMeta") CatalogPO catalogPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET catalog_name = #{newCatalogMeta.catalogName}," - + " metalake_id = #{newCatalogMeta.metalakeId}," - + " type = #{newCatalogMeta.type}," - + " provider = #{newCatalogMeta.provider}," - + " catalog_comment = #{newCatalogMeta.catalogComment}," - + " properties = #{newCatalogMeta.properties}," - + " audit_info = #{newCatalogMeta.auditInfo}," - + " current_version = #{newCatalogMeta.currentVersion}," - + " last_version = #{newCatalogMeta.lastVersion}," - + " deleted_at = #{newCatalogMeta.deletedAt}" - + " WHERE catalog_id = #{oldCatalogMeta.catalogId}" - + " AND catalog_name = #{oldCatalogMeta.catalogName}" - + " AND metalake_id = #{oldCatalogMeta.metalakeId}" - + " AND type = #{oldCatalogMeta.type}" - + " AND provider = #{oldCatalogMeta.provider}" - + " AND (catalog_comment = #{oldCatalogMeta.catalogComment} " - + " OR (catalog_comment IS NULL and #{oldCatalogMeta.catalogComment} IS NULL))" - + " AND properties = #{oldCatalogMeta.properties}" - + " AND audit_info = #{oldCatalogMeta.auditInfo}" - + " AND current_version = #{oldCatalogMeta.currentVersion}" - + " AND last_version = #{oldCatalogMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = CatalogMetaSQLProviderFactory.class, method = "updateCatalogMeta") Integer updateCatalogMeta( @Param("newCatalogMeta") CatalogPO newCatalogPO, @Param("oldCatalogMeta") CatalogPO oldCatalogPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "softDeleteCatalogMetasByCatalogId") Integer softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "softDeleteCatalogMetasByMetalakeId") Integer softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "deleteCatalogMetasByLegacyTimeline") Integer deleteCatalogMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java new file mode 100644 index 00000000000..5c0e63f531b --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.CatalogPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class CatalogMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new CatalogMetaMySQLProvider(), + JDBCBackendType.H2, new CatalogMetaH2Provider()); + + public static CatalogMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class CatalogMetaMySQLProvider extends CatalogMetaBaseSQLProvider {} + + static class CatalogMetaH2Provider extends CatalogMetaBaseSQLProvider {} + + public static String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().listCatalogPOsByMetalakeId(metalakeId); + } + + public static String selectCatalogIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return getProvider().selectCatalogIdByMetalakeIdAndName(metalakeId, name); + } + + public static String selectCatalogMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return getProvider().selectCatalogMetaByMetalakeIdAndName(metalakeId, name); + } + + public static String selectCatalogMetaById(@Param("catalogId") Long catalogId) { + return getProvider().selectCatalogMetaById(catalogId); + } + + public static String insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO) { + return getProvider().insertCatalogMeta(catalogPO); + } + + public static String insertCatalogMetaOnDuplicateKeyUpdate( + @Param("catalogMeta") CatalogPO catalogPO) { + return getProvider().insertCatalogMetaOnDuplicateKeyUpdate(catalogPO); + } + + public static String updateCatalogMeta( + @Param("newCatalogMeta") CatalogPO newCatalogPO, + @Param("oldCatalogMeta") CatalogPO oldCatalogPO) { + return getProvider().updateCatalogMeta(newCatalogPO, oldCatalogPO); + } + + public static String softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteCatalogMetasByCatalogId(catalogId); + } + + public static String softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteCatalogMetasByMetalakeId(metalakeId); + } + + public static String deleteCatalogMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteCatalogMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaBaseSQLProvider.java new file mode 100644 index 00000000000..fc79e6d8ead --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaBaseSQLProvider.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper.META_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper.VERSION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.FilesetPO; +import org.apache.ibatis.annotations.Param; + +public class FilesetMetaBaseSQLProvider { + + public String listFilesetPOsBySchemaId(@Param("schemaId") Long schemaId) { + return "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.schema_id = #{schemaId} AND fm.deleted_at = 0 AND vi.deleted_at = 0"; + } + + public String selectFilesetIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return "SELECT fileset_id as filesetId FROM " + + META_TABLE_NAME + + " WHERE schema_id = #{schemaId} AND fileset_name = #{filesetName}" + + " AND deleted_at = 0"; + } + + public String selectFilesetMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.schema_id = #{schemaId} AND fm.fileset_name = #{filesetName}" + + " AND fm.deleted_at = 0 AND vi.deleted_at = 0"; + } + + public String selectFilesetMetaById(@Param("filesetId") Long filesetId) { + return "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.fileset_id = #{filesetId}" + + " AND fm.deleted_at = 0 AND vi.deleted_at = 0"; + } + + public String insertFilesetMeta(@Param("filesetMeta") FilesetPO filesetPO) { + return "INSERT INTO " + + META_TABLE_NAME + + "(fileset_id, fileset_name, metalake_id," + + " catalog_id, schema_id, type, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{filesetMeta.filesetId}," + + " #{filesetMeta.filesetName}," + + " #{filesetMeta.metalakeId}," + + " #{filesetMeta.catalogId}," + + " #{filesetMeta.schemaId}," + + " #{filesetMeta.type}," + + " #{filesetMeta.auditInfo}," + + " #{filesetMeta.currentVersion}," + + " #{filesetMeta.lastVersion}," + + " #{filesetMeta.deletedAt}" + + " )"; + } + + public String insertFilesetMetaOnDuplicateKeyUpdate(@Param("filesetMeta") FilesetPO filesetPO) { + return "INSERT INTO " + + META_TABLE_NAME + + "(fileset_id, fileset_name, metalake_id," + + " catalog_id, schema_id, type, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{filesetMeta.filesetId}," + + " #{filesetMeta.filesetName}," + + " #{filesetMeta.metalakeId}," + + " #{filesetMeta.catalogId}," + + " #{filesetMeta.schemaId}," + + " #{filesetMeta.type}," + + " #{filesetMeta.auditInfo}," + + " #{filesetMeta.currentVersion}," + + " #{filesetMeta.lastVersion}," + + " #{filesetMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " fileset_name = #{filesetMeta.filesetName}," + + " metalake_id = #{filesetMeta.metalakeId}," + + " catalog_id = #{filesetMeta.catalogId}," + + " schema_id = #{filesetMeta.schemaId}," + + " type = #{filesetMeta.type}," + + " audit_info = #{filesetMeta.auditInfo}," + + " current_version = #{filesetMeta.currentVersion}," + + " last_version = #{filesetMeta.lastVersion}," + + " deleted_at = #{filesetMeta.deletedAt}"; + } + + public String updateFilesetMeta( + @Param("newFilesetMeta") FilesetPO newFilesetPO, + @Param("oldFilesetMeta") FilesetPO oldFilesetPO) { + return "UPDATE " + + META_TABLE_NAME + + " SET fileset_name = #{newFilesetMeta.filesetName}," + + " metalake_id = #{newFilesetMeta.metalakeId}," + + " catalog_id = #{newFilesetMeta.catalogId}," + + " schema_id = #{newFilesetMeta.schemaId}," + + " type = #{newFilesetMeta.type}," + + " audit_info = #{newFilesetMeta.auditInfo}," + + " current_version = #{newFilesetMeta.currentVersion}," + + " last_version = #{newFilesetMeta.lastVersion}," + + " deleted_at = #{newFilesetMeta.deletedAt}" + + " WHERE fileset_id = #{oldFilesetMeta.filesetId}" + + " AND fileset_name = #{oldFilesetMeta.filesetName}" + + " AND metalake_id = #{oldFilesetMeta.metalakeId}" + + " AND catalog_id = #{oldFilesetMeta.catalogId}" + + " AND schema_id = #{oldFilesetMeta.schemaId}" + + " AND type = #{oldFilesetMeta.type}" + + " AND audit_info = #{oldFilesetMeta.auditInfo}" + + " AND current_version = #{oldFilesetMeta.currentVersion}" + + " AND last_version = #{oldFilesetMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + public String deleteFilesetMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + META_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java index 61e4d2f95ff..8692f8f890d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java @@ -21,13 +21,13 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.FilesetPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Result; import org.apache.ibatis.annotations.Results; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for fileset meta operation SQLs. @@ -42,19 +42,6 @@ public interface FilesetMetaMapper { String VERSION_TABLE_NAME = "fileset_version_info"; - @Select( - "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," - + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," - + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," - + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," - + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," - + " vi.deleted_at as version_deleted_at" - + " FROM " - + META_TABLE_NAME - + " fm INNER JOIN " - + VERSION_TABLE_NAME - + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" - + " WHERE fm.schema_id = #{schemaId} AND fm.deleted_at = 0 AND vi.deleted_at = 0") @Results({ @Result(property = "filesetId", column = "fileset_id"), @Result(property = "filesetName", column = "fileset_name"), @@ -77,30 +64,15 @@ public interface FilesetMetaMapper { @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") }) + @SelectProvider(type = FilesetMetaSQLProviderFactory.class, method = "listFilesetPOsBySchemaId") List listFilesetPOsBySchemaId(@Param("schemaId") Long schemaId); - @Select( - "SELECT fileset_id as filesetId FROM " - + META_TABLE_NAME - + " WHERE schema_id = #{schemaId} AND fileset_name = #{filesetName}" - + " AND deleted_at = 0") + @SelectProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "selectFilesetIdBySchemaIdAndName") Long selectFilesetIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("filesetName") String name); - @Select( - "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," - + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," - + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," - + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," - + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," - + " vi.deleted_at as version_deleted_at" - + " FROM " - + META_TABLE_NAME - + " fm INNER JOIN " - + VERSION_TABLE_NAME - + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" - + " WHERE fm.schema_id = #{schemaId} AND fm.fileset_name = #{filesetName}" - + " AND fm.deleted_at = 0 AND vi.deleted_at = 0") @Results({ @Result(property = "filesetId", column = "fileset_id"), @Result(property = "filesetName", column = "fileset_name"), @@ -123,23 +95,12 @@ Long selectFilesetIdBySchemaIdAndName( @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") }) + @SelectProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "selectFilesetMetaBySchemaIdAndName") FilesetPO selectFilesetMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("filesetName") String name); - @Select( - "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," - + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," - + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," - + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," - + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," - + " vi.deleted_at as version_deleted_at" - + " FROM " - + META_TABLE_NAME - + " fm INNER JOIN " - + VERSION_TABLE_NAME - + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" - + " WHERE fm.fileset_id = #{filesetId}" - + " AND fm.deleted_at = 0 AND vi.deleted_at = 0") @Results({ @Result(property = "filesetId", column = "fileset_id"), @Result(property = "filesetName", column = "fileset_name"), @@ -162,120 +123,45 @@ FilesetPO selectFilesetMetaBySchemaIdAndName( @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") }) + @SelectProvider(type = FilesetMetaSQLProviderFactory.class, method = "selectFilesetMetaById") FilesetPO selectFilesetMetaById(@Param("filesetId") Long filesetId); - @Insert( - "INSERT INTO " - + META_TABLE_NAME - + "(fileset_id, fileset_name, metalake_id," - + " catalog_id, schema_id, type, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{filesetMeta.filesetId}," - + " #{filesetMeta.filesetName}," - + " #{filesetMeta.metalakeId}," - + " #{filesetMeta.catalogId}," - + " #{filesetMeta.schemaId}," - + " #{filesetMeta.type}," - + " #{filesetMeta.auditInfo}," - + " #{filesetMeta.currentVersion}," - + " #{filesetMeta.lastVersion}," - + " #{filesetMeta.deletedAt}" - + " )") + @InsertProvider(type = FilesetMetaSQLProviderFactory.class, method = "insertFilesetMeta") void insertFilesetMeta(@Param("filesetMeta") FilesetPO filesetPO); - @Insert( - "INSERT INTO " - + META_TABLE_NAME - + "(fileset_id, fileset_name, metalake_id," - + " catalog_id, schema_id, type, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{filesetMeta.filesetId}," - + " #{filesetMeta.filesetName}," - + " #{filesetMeta.metalakeId}," - + " #{filesetMeta.catalogId}," - + " #{filesetMeta.schemaId}," - + " #{filesetMeta.type}," - + " #{filesetMeta.auditInfo}," - + " #{filesetMeta.currentVersion}," - + " #{filesetMeta.lastVersion}," - + " #{filesetMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " fileset_name = #{filesetMeta.filesetName}," - + " metalake_id = #{filesetMeta.metalakeId}," - + " catalog_id = #{filesetMeta.catalogId}," - + " schema_id = #{filesetMeta.schemaId}," - + " type = #{filesetMeta.type}," - + " audit_info = #{filesetMeta.auditInfo}," - + " current_version = #{filesetMeta.currentVersion}," - + " last_version = #{filesetMeta.lastVersion}," - + " deleted_at = #{filesetMeta.deletedAt}") + @InsertProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "insertFilesetMetaOnDuplicateKeyUpdate") void insertFilesetMetaOnDuplicateKeyUpdate(@Param("filesetMeta") FilesetPO filesetPO); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET fileset_name = #{newFilesetMeta.filesetName}," - + " metalake_id = #{newFilesetMeta.metalakeId}," - + " catalog_id = #{newFilesetMeta.catalogId}," - + " schema_id = #{newFilesetMeta.schemaId}," - + " type = #{newFilesetMeta.type}," - + " audit_info = #{newFilesetMeta.auditInfo}," - + " current_version = #{newFilesetMeta.currentVersion}," - + " last_version = #{newFilesetMeta.lastVersion}," - + " deleted_at = #{newFilesetMeta.deletedAt}" - + " WHERE fileset_id = #{oldFilesetMeta.filesetId}" - + " AND fileset_name = #{oldFilesetMeta.filesetName}" - + " AND metalake_id = #{oldFilesetMeta.metalakeId}" - + " AND catalog_id = #{oldFilesetMeta.catalogId}" - + " AND schema_id = #{oldFilesetMeta.schemaId}" - + " AND type = #{oldFilesetMeta.type}" - + " AND audit_info = #{oldFilesetMeta.auditInfo}" - + " AND current_version = #{oldFilesetMeta.currentVersion}" - + " AND last_version = #{oldFilesetMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = FilesetMetaSQLProviderFactory.class, method = "updateFilesetMeta") Integer updateFilesetMeta( @Param("newFilesetMeta") FilesetPO newFilesetPO, @Param("oldFilesetMeta") FilesetPO oldFilesetPO); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasByMetalakeId") Integer softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasByCatalogId") Integer softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasBySchemaId") Integer softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasByFilesetId") Integer softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId); - @Delete( - "DELETE FROM " - + META_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "deleteFilesetMetasByLegacyTimeline") Integer deleteFilesetMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java new file mode 100644 index 00000000000..36ea94d5862 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.FilesetPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class FilesetMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new FilesetMetaMySQLProvider(), + JDBCBackendType.H2, new FilesetMetaH2Provider()); + + public static FilesetMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class FilesetMetaMySQLProvider extends FilesetMetaBaseSQLProvider {} + + static class FilesetMetaH2Provider extends FilesetMetaBaseSQLProvider {} + + public static String listFilesetPOsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().listFilesetPOsBySchemaId(schemaId); + } + + public static String selectFilesetIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return getProvider().selectFilesetIdBySchemaIdAndName(schemaId, name); + } + + public static String selectFilesetMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return getProvider().selectFilesetMetaBySchemaIdAndName(schemaId, name); + } + + public static String selectFilesetMetaById(@Param("filesetId") Long filesetId) { + return getProvider().selectFilesetMetaById(filesetId); + } + + public static String insertFilesetMeta(@Param("filesetMeta") FilesetPO filesetPO) { + return getProvider().insertFilesetMeta(filesetPO); + } + + public static String insertFilesetMetaOnDuplicateKeyUpdate( + @Param("filesetMeta") FilesetPO filesetPO) { + return getProvider().insertFilesetMetaOnDuplicateKeyUpdate(filesetPO); + } + + public static String updateFilesetMeta( + @Param("newFilesetMeta") FilesetPO newFilesetPO, + @Param("oldFilesetMeta") FilesetPO oldFilesetPO) { + return getProvider().updateFilesetMeta(newFilesetPO, oldFilesetPO); + } + + public static String softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteFilesetMetasByMetalakeId(metalakeId); + } + + public static String softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteFilesetMetasByCatalogId(catalogId); + } + + public static String softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteFilesetMetasBySchemaId(schemaId); + } + + public String softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId) { + return getProvider().softDeleteFilesetMetasByFilesetId(filesetId); + } + + public String deleteFilesetMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteFilesetMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionBaseSQLProvider.java new file mode 100644 index 00000000000..f6ab85b38be --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionBaseSQLProvider.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper.VERSION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.FilesetVersionPO; +import org.apache.ibatis.annotations.Param; + +public class FilesetVersionBaseSQLProvider { + public String insertFilesetVersion(@Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return "INSERT INTO " + + VERSION_TABLE_NAME + + "(metalake_id, catalog_id, schema_id, fileset_id," + + " version, fileset_comment, properties, storage_location," + + " deleted_at)" + + " VALUES(" + + " #{filesetVersion.metalakeId}," + + " #{filesetVersion.catalogId}," + + " #{filesetVersion.schemaId}," + + " #{filesetVersion.filesetId}," + + " #{filesetVersion.version}," + + " #{filesetVersion.filesetComment}," + + " #{filesetVersion.properties}," + + " #{filesetVersion.storageLocation}," + + " #{filesetVersion.deletedAt}" + + " )"; + } + + public String insertFilesetVersionOnDuplicateKeyUpdate( + @Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return "INSERT INTO " + + VERSION_TABLE_NAME + + "(metalake_id, catalog_id, schema_id, fileset_id," + + " version, fileset_comment, properties, storage_location," + + " deleted_at)" + + " VALUES(" + + " #{filesetVersion.metalakeId}," + + " #{filesetVersion.catalogId}," + + " #{filesetVersion.schemaId}," + + " #{filesetVersion.filesetId}," + + " #{filesetVersion.version}," + + " #{filesetVersion.filesetComment}," + + " #{filesetVersion.properties}," + + " #{filesetVersion.storageLocation}," + + " #{filesetVersion.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " metalake_id = #{filesetVersion.metalakeId}," + + " catalog_id = #{filesetVersion.catalogId}," + + " schema_id = #{filesetVersion.schemaId}," + + " fileset_id = #{filesetVersion.filesetId}," + + " version = #{filesetVersion.version}," + + " fileset_comment = #{filesetVersion.filesetComment}," + + " properties = #{filesetVersion.properties}," + + " storage_location = #{filesetVersion.storageLocation}," + + " deleted_at = #{filesetVersion.deletedAt}"; + } + + public String softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + public String deleteFilesetVersionsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + VERSION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } + + public String selectFilesetVersionsByRetentionCount( + @Param("versionRetentionCount") Long versionRetentionCount) { + return "SELECT fileset_id as filesetId," + + " Max(version) as version" + + " FROM " + + VERSION_TABLE_NAME + + " WHERE version > #{versionRetentionCount} AND deleted_at = 0" + + " GROUP BY fileset_id"; + } + + public String softDeleteFilesetVersionsByRetentionLine( + @Param("filesetId") Long filesetId, + @Param("versionRetentionLine") long versionRetentionLine, + @Param("limit") int limit) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java index cd4bfb9b613..09eca4b90e0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java @@ -22,11 +22,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.FilesetMaxVersionPO; import org.apache.gravitino.storage.relational.po.FilesetVersionPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for fileset version info operation SQLs. @@ -39,110 +39,50 @@ public interface FilesetVersionMapper { String VERSION_TABLE_NAME = "fileset_version_info"; - @Insert( - "INSERT INTO " - + VERSION_TABLE_NAME - + "(metalake_id, catalog_id, schema_id, fileset_id," - + " version, fileset_comment, properties, storage_location," - + " deleted_at)" - + " VALUES(" - + " #{filesetVersion.metalakeId}," - + " #{filesetVersion.catalogId}," - + " #{filesetVersion.schemaId}," - + " #{filesetVersion.filesetId}," - + " #{filesetVersion.version}," - + " #{filesetVersion.filesetComment}," - + " #{filesetVersion.properties}," - + " #{filesetVersion.storageLocation}," - + " #{filesetVersion.deletedAt}" - + " )") + @InsertProvider(type = FilesetVersionSQLProviderFactory.class, method = "insertFilesetVersion") void insertFilesetVersion(@Param("filesetVersion") FilesetVersionPO filesetVersionPO); - @Insert( - "INSERT INTO " - + VERSION_TABLE_NAME - + "(metalake_id, catalog_id, schema_id, fileset_id," - + " version, fileset_comment, properties, storage_location," - + " deleted_at)" - + " VALUES(" - + " #{filesetVersion.metalakeId}," - + " #{filesetVersion.catalogId}," - + " #{filesetVersion.schemaId}," - + " #{filesetVersion.filesetId}," - + " #{filesetVersion.version}," - + " #{filesetVersion.filesetComment}," - + " #{filesetVersion.properties}," - + " #{filesetVersion.storageLocation}," - + " #{filesetVersion.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " metalake_id = #{filesetVersion.metalakeId}," - + " catalog_id = #{filesetVersion.catalogId}," - + " schema_id = #{filesetVersion.schemaId}," - + " fileset_id = #{filesetVersion.filesetId}," - + " version = #{filesetVersion.version}," - + " fileset_comment = #{filesetVersion.filesetComment}," - + " properties = #{filesetVersion.properties}," - + " storage_location = #{filesetVersion.storageLocation}," - + " deleted_at = #{filesetVersion.deletedAt}") + @InsertProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "insertFilesetVersionOnDuplicateKeyUpdate") void insertFilesetVersionOnDuplicateKeyUpdate( @Param("filesetVersion") FilesetVersionPO filesetVersionPO); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByMetalakeId") Integer softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByCatalogId") Integer softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsBySchemaId") Integer softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByFilesetId") Integer softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId); - @Delete( - "DELETE FROM " - + VERSION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "deleteFilesetVersionsByLegacyTimeline") Integer deleteFilesetVersionsByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); - @Select( - "SELECT fileset_id as filesetId," - + " Max(version) as version" - + " FROM " - + VERSION_TABLE_NAME - + " WHERE version > #{versionRetentionCount} AND deleted_at = 0" - + " GROUP BY fileset_id") + @SelectProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "selectFilesetVersionsByRetentionCount") List selectFilesetVersionsByRetentionCount( @Param("versionRetentionCount") Long versionRetentionCount); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByRetentionLine") Integer softDeleteFilesetVersionsByRetentionLine( @Param("filesetId") Long filesetId, @Param("versionRetentionLine") long versionRetentionLine, diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java new file mode 100644 index 00000000000..163f2c882fe --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java @@ -0,0 +1,93 @@ +package org.apache.gravitino.storage.relational.mapper; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.FilesetVersionPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class FilesetVersionSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new FilesetVersionMySQLProvider(), + JDBCBackendType.H2, new FilesetVersionH2Provider()); + + public static FilesetVersionBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class FilesetVersionMySQLProvider extends FilesetVersionBaseSQLProvider {} + + static class FilesetVersionH2Provider extends FilesetVersionBaseSQLProvider {} + + public static String insertFilesetVersion( + @Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return getProvider().insertFilesetVersion(filesetVersionPO); + } + + public static String insertFilesetVersionOnDuplicateKeyUpdate( + @Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return getProvider().insertFilesetVersionOnDuplicateKeyUpdate(filesetVersionPO); + } + + public static String softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteFilesetVersionsByMetalakeId(metalakeId); + } + + public static String softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteFilesetVersionsByCatalogId(catalogId); + } + + public static String softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteFilesetVersionsBySchemaId(schemaId); + } + + public static String softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId) { + return getProvider().softDeleteFilesetVersionsByFilesetId(filesetId); + } + + public static String deleteFilesetVersionsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteFilesetVersionsByLegacyTimeline(legacyTimeline, limit); + } + + public static String selectFilesetVersionsByRetentionCount( + @Param("versionRetentionCount") Long versionRetentionCount) { + return getProvider().selectFilesetVersionsByRetentionCount(versionRetentionCount); + } + + public static String softDeleteFilesetVersionsByRetentionLine( + @Param("filesetId") Long filesetId, + @Param("versionRetentionLine") long versionRetentionLine, + @Param("limit") int limit) { + return getProvider() + .softDeleteFilesetVersionsByRetentionLine(filesetId, versionRetentionLine, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaBaseSQLProvider.java new file mode 100644 index 00000000000..a782ddb8f0f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaBaseSQLProvider.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.ibatis.annotations.Param; + +public class MetalakeMetaBaseSQLProvider { + + public String listMetalakePOs() { + return "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE deleted_at = 0"; + } + + public String selectMetalakeMetaByName(@Param("metalakeName") String metalakeName) { + return "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_name = #{metalakeName} and deleted_at = 0"; + } + + public String selectMetalakeMetaById(@Param("metalakeId") Long metalakeId) { + return "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} and deleted_at = 0"; + } + + public String selectMetalakeIdMetaByName(@Param("metalakeName") String metalakeName) { + return "SELECT metalake_id as metalakeId" + + " FROM " + + TABLE_NAME + + " WHERE metalake_name = #{metalakeName} and deleted_at = 0"; + } + + public String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) { + return "INSERT INTO " + + TABLE_NAME + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{metalakeMeta.metalakeId}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + + " )"; + } + + public String insertMetalakeMetaOnDuplicateKeyUpdate( + @Param("metalakeMeta") MetalakePO metalakePO) { + return "INSERT INTO " + + TABLE_NAME + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{metalakeMeta.metalakeId}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}," + + " current_version = #{metalakeMeta.currentVersion}," + + " last_version = #{metalakeMeta.lastVersion}," + + " deleted_at = #{metalakeMeta.deletedAt}"; + } + + public String updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO) { + return "UPDATE " + + TABLE_NAME + + " SET metalake_name = #{newMetalakeMeta.metalakeName}," + + " metalake_comment = #{newMetalakeMeta.metalakeComment}," + + " properties = #{newMetalakeMeta.properties}," + + " audit_info = #{newMetalakeMeta.auditInfo}," + + " schema_version = #{newMetalakeMeta.schemaVersion}," + + " current_version = #{newMetalakeMeta.currentVersion}," + + " last_version = #{newMetalakeMeta.lastVersion}" + + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" + + " AND metalake_name = #{oldMetalakeMeta.metalakeName}" + + " AND (metalake_comment = #{oldMetalakeMeta.metalakeComment} " + + " OR (metalake_comment IS NULL and #{oldMetalakeMeta.metalakeComment} IS NULL))" + + " AND properties = #{oldMetalakeMeta.properties}" + + " AND audit_info = #{oldMetalakeMeta.auditInfo}" + + " AND schema_version = #{oldMetalakeMeta.schemaVersion}" + + " AND current_version = #{oldMetalakeMeta.currentVersion}" + + " AND last_version = #{oldMetalakeMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteMetalakeMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index 89f8d13ceb1..d5dc809bfe2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.MetalakePO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for metalake meta operation SQLs. @@ -38,126 +38,41 @@ public interface MetalakeMetaMapper { String TABLE_NAME = "metalake_meta"; - @Select( - "SELECT metalake_id as metalakeId, metalake_name as metalakeName," - + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE deleted_at = 0") + @SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "listMetalakePOs") List listMetalakePOs(); - @Select( - "SELECT metalake_id as metalakeId, metalake_name as metalakeName," - + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_name = #{metalakeName} and deleted_at = 0") + @SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeMetaByName") MetalakePO selectMetalakeMetaByName(@Param("metalakeName") String name); - @Select( - "SELECT metalake_id as metalakeId, metalake_name as metalakeName," - + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} and deleted_at = 0") + @SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeMetaById") MetalakePO selectMetalakeMetaById(@Param("metalakeId") Long metalakeId); - @Select( - "SELECT metalake_id FROM " - + TABLE_NAME - + " WHERE metalake_name = #{metalakeName} and deleted_at = 0") + @SelectProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "selectMetalakeIdMetaByName") Long selectMetalakeIdMetaByName(@Param("metalakeName") String name); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," - + " schema_version, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{metalakeMeta.metalakeId}," - + " #{metalakeMeta.metalakeName}," - + " #{metalakeMeta.metalakeComment}," - + " #{metalakeMeta.properties}," - + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}," - + " #{metalakeMeta.currentVersion}," - + " #{metalakeMeta.lastVersion}," - + " #{metalakeMeta.deletedAt}" - + " )") + @InsertProvider(type = MetalakeMetaSQLProviderFactory.class, method = "insertMetalakeMeta") void insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," - + " schema_version, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{metalakeMeta.metalakeId}," - + " #{metalakeMeta.metalakeName}," - + " #{metalakeMeta.metalakeComment}," - + " #{metalakeMeta.properties}," - + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}," - + " #{metalakeMeta.currentVersion}," - + " #{metalakeMeta.lastVersion}," - + " #{metalakeMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " metalake_name = #{metalakeMeta.metalakeName}," - + " metalake_comment = #{metalakeMeta.metalakeComment}," - + " properties = #{metalakeMeta.properties}," - + " audit_info = #{metalakeMeta.auditInfo}," - + " schema_version = #{metalakeMeta.schemaVersion}," - + " current_version = #{metalakeMeta.currentVersion}," - + " last_version = #{metalakeMeta.lastVersion}," - + " deleted_at = #{metalakeMeta.deletedAt}") + @InsertProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "insertMetalakeMetaOnDuplicateKeyUpdate") void insertMetalakeMetaOnDuplicateKeyUpdate(@Param("metalakeMeta") MetalakePO metalakePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET metalake_name = #{newMetalakeMeta.metalakeName}," - + " metalake_comment = #{newMetalakeMeta.metalakeComment}," - + " properties = #{newMetalakeMeta.properties}," - + " audit_info = #{newMetalakeMeta.auditInfo}," - + " schema_version = #{newMetalakeMeta.schemaVersion}," - + " current_version = #{newMetalakeMeta.currentVersion}," - + " last_version = #{newMetalakeMeta.lastVersion}" - + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" - + " AND metalake_name = #{oldMetalakeMeta.metalakeName}" - + " AND (metalake_comment = #{oldMetalakeMeta.metalakeComment} " - + " OR (metalake_comment IS NULL and #{oldMetalakeMeta.metalakeComment} IS NULL))" - + " AND properties = #{oldMetalakeMeta.properties}" - + " AND audit_info = #{oldMetalakeMeta.auditInfo}" - + " AND schema_version = #{oldMetalakeMeta.schemaVersion}" - + " AND current_version = #{oldMetalakeMeta.currentVersion}" - + " AND last_version = #{oldMetalakeMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = MetalakeMetaSQLProviderFactory.class, method = "updateMetalakeMeta") Integer updateMetalakeMeta( @Param("newMetalakeMeta") MetalakePO newMetalakePO, @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "softDeleteMetalakeMetaByMetalakeId") Integer softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "deleteMetalakeMetasByLegacyTimeline") Integer deleteMetalakeMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java new file mode 100644 index 00000000000..e28cbc9d774 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +/** SQL Provider for Metalake Meta operations. */ +public class MetalakeMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new MetalakeMetaMySQLProvider(), + JDBCBackendType.H2, new MetalakeMetaH2Provider()); + + public static MetalakeMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class MetalakeMetaMySQLProvider extends MetalakeMetaBaseSQLProvider {} + + static class MetalakeMetaH2Provider extends MetalakeMetaBaseSQLProvider {} + + public String listMetalakePOs() { + return getProvider().listMetalakePOs(); + } + + public static String selectMetalakeMetaByName(@Param("metalakeName") String metalakeName) { + return getProvider().selectMetalakeMetaByName(metalakeName); + } + + public static String selectMetalakeMetaById(@Param("metalakeId") Long metalakeId) { + return getProvider().selectMetalakeMetaById(metalakeId); + } + + public static String selectMetalakeIdMetaByName(@Param("metalakeName") String metalakeName) { + return getProvider().selectMetalakeIdMetaByName(metalakeName); + } + + public static String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) { + return getProvider().insertMetalakeMeta(metalakePO); + } + + public static String insertMetalakeMetaOnDuplicateKeyUpdate( + @Param("metalakeMeta") MetalakePO metalakePO) { + return getProvider().insertMetalakeMetaOnDuplicateKeyUpdate(metalakePO); + } + + public static String updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO) { + return getProvider().updateMetalakeMeta(newMetalakePO, oldMetalakePO); + } + + public static String softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteMetalakeMetaByMetalakeId(metalakeId); + } + + public static String deleteMetalakeMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteMetalakeMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaBaseSQLProvider.java new file mode 100644 index 00000000000..056b8d2b581 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaBaseSQLProvider.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.SchemaPO; +import org.apache.ibatis.annotations.Param; + +public class SchemaMetaBaseSQLProvider { + public String listSchemaPOsByCatalogId(@Param("catalogId") Long catalogId) { + return "SELECT schema_id as schemaId, schema_name as schemaName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_comment as schemaComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String selectSchemaIdByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return "SELECT schema_id as schemaId FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName}" + + " AND deleted_at = 0"; + } + + public String selectSchemaMetaByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return "SELECT schema_id as schemaId, schema_name as schemaName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_comment as schemaComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName} AND deleted_at = 0"; + } + + public String selectSchemaMetaById(@Param("schemaId") Long schemaId) { + return "SELECT schema_id as schemaId, schema_name as schemaName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_comment as schemaComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String insertSchemaMeta(@Param("schemaMeta") SchemaPO schemaPO) { + return "INSERT INTO " + + TABLE_NAME + + "(schema_id, schema_name, metalake_id," + + " catalog_id, schema_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{schemaMeta.schemaId}," + + " #{schemaMeta.schemaName}," + + " #{schemaMeta.metalakeId}," + + " #{schemaMeta.catalogId}," + + " #{schemaMeta.schemaComment}," + + " #{schemaMeta.properties}," + + " #{schemaMeta.auditInfo}," + + " #{schemaMeta.currentVersion}," + + " #{schemaMeta.lastVersion}," + + " #{schemaMeta.deletedAt}" + + " )"; + } + + public String insertSchemaMetaOnDuplicateKeyUpdate(@Param("schemaMeta") SchemaPO schemaPO) { + return "INSERT INTO " + + TABLE_NAME + + "(schema_id, schema_name, metalake_id," + + " catalog_id, schema_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{schemaMeta.schemaId}," + + " #{schemaMeta.schemaName}," + + " #{schemaMeta.metalakeId}," + + " #{schemaMeta.catalogId}," + + " #{schemaMeta.schemaComment}," + + " #{schemaMeta.properties}," + + " #{schemaMeta.auditInfo}," + + " #{schemaMeta.currentVersion}," + + " #{schemaMeta.lastVersion}," + + " #{schemaMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " schema_name = #{schemaMeta.schemaName}," + + " metalake_id = #{schemaMeta.metalakeId}," + + " catalog_id = #{schemaMeta.catalogId}," + + " schema_comment = #{schemaMeta.schemaComment}," + + " properties = #{schemaMeta.properties}," + + " audit_info = #{schemaMeta.auditInfo}," + + " current_version = #{schemaMeta.currentVersion}," + + " last_version = #{schemaMeta.lastVersion}," + + " deleted_at = #{schemaMeta.deletedAt}"; + } + + public String updateSchemaMeta( + @Param("newSchemaMeta") SchemaPO newSchemaPO, @Param("oldSchemaMeta") SchemaPO oldSchemaPO) { + return "UPDATE " + + TABLE_NAME + + " SET schema_name = #{newSchemaMeta.schemaName}," + + " metalake_id = #{newSchemaMeta.metalakeId}," + + " catalog_id = #{newSchemaMeta.catalogId}," + + " schema_comment = #{newSchemaMeta.schemaComment}," + + " properties = #{newSchemaMeta.properties}," + + " audit_info = #{newSchemaMeta.auditInfo}," + + " current_version = #{newSchemaMeta.currentVersion}," + + " last_version = #{newSchemaMeta.lastVersion}," + + " deleted_at = #{newSchemaMeta.deletedAt}" + + " WHERE schema_id = #{oldSchemaMeta.schemaId}" + + " AND schema_name = #{oldSchemaMeta.schemaName}" + + " AND metalake_id = #{oldSchemaMeta.metalakeId}" + + " AND catalog_id = #{oldSchemaMeta.catalogId}" + + " AND (schema_comment IS NULL OR schema_comment = #{oldSchemaMeta.schemaComment})" + + " AND properties = #{oldSchemaMeta.properties}" + + " AND audit_info = #{oldSchemaMeta.auditInfo}" + + " AND current_version = #{oldSchemaMeta.currentVersion}" + + " AND last_version = #{oldSchemaMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteSchemaMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String softDeleteSchemaMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String deleteSchemaMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java index a71a4b7b6f3..054248dac0c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.SchemaPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for schema meta operation SQLs. @@ -38,151 +38,54 @@ public interface SchemaMetaMapper { String TABLE_NAME = "schema_meta"; - @Select( - "SELECT schema_id as schemaId, schema_name as schemaName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_comment as schemaComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @SelectProvider(type = SchemaMetaSQLProviderFactory.class, method = "listSchemaPOsByCatalogId") List listSchemaPOsByCatalogId(@Param("catalogId") Long catalogId); - @Select( - "SELECT schema_id as schemaId FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName}" - + " AND deleted_at = 0") + @SelectProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "selectSchemaIdByCatalogIdAndName") Long selectSchemaIdByCatalogIdAndName( @Param("catalogId") Long catalogId, @Param("schemaName") String name); - @Select( - "SELECT schema_id as schemaId, schema_name as schemaName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_comment as schemaComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName} AND deleted_at = 0") + @SelectProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "selectSchemaMetaByCatalogIdAndName") SchemaPO selectSchemaMetaByCatalogIdAndName( @Param("catalogId") Long catalogId, @Param("schemaName") String name); - @Select( - "SELECT schema_id as schemaId, schema_name as schemaName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_comment as schemaComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @SelectProvider(type = SchemaMetaSQLProviderFactory.class, method = "selectSchemaMetaById") SchemaPO selectSchemaMetaById(@Param("schemaId") Long schemaId); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(schema_id, schema_name, metalake_id," - + " catalog_id, schema_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{schemaMeta.schemaId}," - + " #{schemaMeta.schemaName}," - + " #{schemaMeta.metalakeId}," - + " #{schemaMeta.catalogId}," - + " #{schemaMeta.schemaComment}," - + " #{schemaMeta.properties}," - + " #{schemaMeta.auditInfo}," - + " #{schemaMeta.currentVersion}," - + " #{schemaMeta.lastVersion}," - + " #{schemaMeta.deletedAt}" - + " )") + @InsertProvider(type = SchemaMetaSQLProviderFactory.class, method = "insertSchemaMeta") void insertSchemaMeta(@Param("schemaMeta") SchemaPO schemaPO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(schema_id, schema_name, metalake_id," - + " catalog_id, schema_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{schemaMeta.schemaId}," - + " #{schemaMeta.schemaName}," - + " #{schemaMeta.metalakeId}," - + " #{schemaMeta.catalogId}," - + " #{schemaMeta.schemaComment}," - + " #{schemaMeta.properties}," - + " #{schemaMeta.auditInfo}," - + " #{schemaMeta.currentVersion}," - + " #{schemaMeta.lastVersion}," - + " #{schemaMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " schema_name = #{schemaMeta.schemaName}," - + " metalake_id = #{schemaMeta.metalakeId}," - + " catalog_id = #{schemaMeta.catalogId}," - + " schema_comment = #{schemaMeta.schemaComment}," - + " properties = #{schemaMeta.properties}," - + " audit_info = #{schemaMeta.auditInfo}," - + " current_version = #{schemaMeta.currentVersion}," - + " last_version = #{schemaMeta.lastVersion}," - + " deleted_at = #{schemaMeta.deletedAt}") + @InsertProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "insertSchemaMetaOnDuplicateKeyUpdate") void insertSchemaMetaOnDuplicateKeyUpdate(@Param("schemaMeta") SchemaPO schemaPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET schema_name = #{newSchemaMeta.schemaName}," - + " metalake_id = #{newSchemaMeta.metalakeId}," - + " catalog_id = #{newSchemaMeta.catalogId}," - + " schema_comment = #{newSchemaMeta.schemaComment}," - + " properties = #{newSchemaMeta.properties}," - + " audit_info = #{newSchemaMeta.auditInfo}," - + " current_version = #{newSchemaMeta.currentVersion}," - + " last_version = #{newSchemaMeta.lastVersion}," - + " deleted_at = #{newSchemaMeta.deletedAt}" - + " WHERE schema_id = #{oldSchemaMeta.schemaId}" - + " AND schema_name = #{oldSchemaMeta.schemaName}" - + " AND metalake_id = #{oldSchemaMeta.metalakeId}" - + " AND catalog_id = #{oldSchemaMeta.catalogId}" - + " AND (schema_comment IS NULL OR schema_comment = #{oldSchemaMeta.schemaComment})" - + " AND properties = #{oldSchemaMeta.properties}" - + " AND audit_info = #{oldSchemaMeta.auditInfo}" - + " AND current_version = #{oldSchemaMeta.currentVersion}" - + " AND last_version = #{oldSchemaMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = SchemaMetaSQLProviderFactory.class, method = "updateSchemaMeta") Integer updateSchemaMeta( @Param("newSchemaMeta") SchemaPO newSchemaPO, @Param("oldSchemaMeta") SchemaPO oldSchemaPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "softDeleteSchemaMetasBySchemaId") Integer softDeleteSchemaMetasBySchemaId(@Param("schemaId") Long schemaId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "softDeleteSchemaMetasByMetalakeId") Integer softDeleteSchemaMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "softDeleteSchemaMetasByCatalogId") Integer softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "deleteSchemaMetasByLegacyTimeline") Integer deleteSchemaMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java new file mode 100644 index 00000000000..5fa6252d5b6 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.SchemaPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class SchemaMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new SchemaMetaMySQLProvider(), + JDBCBackendType.H2, new SchemaMetaH2Provider()); + + public static SchemaMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class SchemaMetaMySQLProvider extends SchemaMetaBaseSQLProvider {} + + static class SchemaMetaH2Provider extends SchemaMetaBaseSQLProvider {} + + public static String listSchemaPOsByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().listSchemaPOsByCatalogId(catalogId); + } + + public static String selectSchemaIdByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return getProvider().selectSchemaIdByCatalogIdAndName(catalogId, name); + } + + public static String selectSchemaMetaByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return getProvider().selectSchemaMetaByCatalogIdAndName(catalogId, name); + } + + public static String selectSchemaMetaById(@Param("schemaId") Long schemaId) { + return getProvider().selectSchemaMetaById(schemaId); + } + + public static String insertSchemaMeta(@Param("schemaMeta") SchemaPO schemaPO) { + return getProvider().insertSchemaMeta(schemaPO); + } + + public static String insertSchemaMetaOnDuplicateKeyUpdate( + @Param("schemaMeta") SchemaPO schemaPO) { + return getProvider().insertSchemaMetaOnDuplicateKeyUpdate(schemaPO); + } + + public static String updateSchemaMeta( + @Param("newSchemaMeta") SchemaPO newSchemaPO, @Param("oldSchemaMeta") SchemaPO oldSchemaPO) { + return getProvider().updateSchemaMeta(newSchemaPO, oldSchemaPO); + } + + public static String softDeleteSchemaMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteSchemaMetasBySchemaId(schemaId); + } + + public static String softDeleteSchemaMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteSchemaMetasByMetalakeId(metalakeId); + } + + public static String softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteSchemaMetasByCatalogId(catalogId); + } + + public static String deleteSchemaMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteSchemaMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaBaseSQLProvider.java new file mode 100644 index 00000000000..03cf2582ff6 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaBaseSQLProvider.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.TablePO; +import org.apache.ibatis.annotations.Param; + +public class TableMetaBaseSQLProvider { + + public String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) { + return "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String selectTableIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return "SELECT table_id as tableId FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND table_name = #{tableName}" + + " AND deleted_at = 0"; + } + + public String selectTableMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND table_name = #{tableName} AND deleted_at = 0"; + } + + public String selectTableMetaById(@Param("tableId") Long tableId) { + return "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE table_id = #{tableId} AND deleted_at = 0"; + } + + public String insertTableMeta(@Param("tableMeta") TablePO tablePO) { + return "INSERT INTO " + + TABLE_NAME + + "(table_id, table_name, metalake_id," + + " catalog_id, schema_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tableMeta.tableId}," + + " #{tableMeta.tableName}," + + " #{tableMeta.metalakeId}," + + " #{tableMeta.catalogId}," + + " #{tableMeta.schemaId}," + + " #{tableMeta.auditInfo}," + + " #{tableMeta.currentVersion}," + + " #{tableMeta.lastVersion}," + + " #{tableMeta.deletedAt}" + + " )"; + } + + public String insertTableMetaOnDuplicateKeyUpdate(@Param("tableMeta") TablePO tablePO) { + return "INSERT INTO " + + TABLE_NAME + + "(table_id, table_name, metalake_id," + + " catalog_id, schema_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tableMeta.tableId}," + + " #{tableMeta.tableName}," + + " #{tableMeta.metalakeId}," + + " #{tableMeta.catalogId}," + + " #{tableMeta.schemaId}," + + " #{tableMeta.auditInfo}," + + " #{tableMeta.currentVersion}," + + " #{tableMeta.lastVersion}," + + " #{tableMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " table_name = #{tableMeta.tableName}," + + " metalake_id = #{tableMeta.metalakeId}," + + " catalog_id = #{tableMeta.catalogId}," + + " schema_id = #{tableMeta.schemaId}," + + " audit_info = #{tableMeta.auditInfo}," + + " current_version = #{tableMeta.currentVersion}," + + " last_version = #{tableMeta.lastVersion}," + + " deleted_at = #{tableMeta.deletedAt}"; + } + + public String updateTableMeta( + @Param("newTableMeta") TablePO newTablePO, @Param("oldTableMeta") TablePO oldTablePO) { + return "UPDATE " + + TABLE_NAME + + " SET table_name = #{newTableMeta.tableName}," + + " metalake_id = #{newTableMeta.metalakeId}," + + " catalog_id = #{newTableMeta.catalogId}," + + " schema_id = #{newTableMeta.schemaId}," + + " audit_info = #{newTableMeta.auditInfo}," + + " current_version = #{newTableMeta.currentVersion}," + + " last_version = #{newTableMeta.lastVersion}," + + " deleted_at = #{newTableMeta.deletedAt}" + + " WHERE table_id = #{oldTableMeta.tableId}" + + " AND table_name = #{oldTableMeta.tableName}" + + " AND metalake_id = #{oldTableMeta.metalakeId}" + + " AND catalog_id = #{oldTableMeta.catalogId}" + + " AND schema_id = #{oldTableMeta.schemaId}" + + " AND audit_info = #{oldTableMeta.auditInfo}" + + " AND current_version = #{oldTableMeta.currentVersion}" + + " AND last_version = #{oldTableMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteTableMetasByTableId(@Param("tableId") Long tableId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE table_id = #{tableId} AND deleted_at = 0"; + } + + public String softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String deleteTableMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java index 98edc595fd7..a5224593765 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TablePO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -38,154 +38,59 @@ public interface TableMetaMapper { String TABLE_NAME = "table_meta"; - @Select( - "SELECT table_id as tableId, table_name as tableName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_id as schemaId, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsBySchemaId") List listTablePOsBySchemaId(@Param("schemaId") Long schemaId); - @Select( - "SELECT table_id as tableId FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND table_name = #{tableName}" - + " AND deleted_at = 0") + @SelectProvider( + type = TableMetaSQLProviderFactory.class, + method = "selectTableIdBySchemaIdAndName") Long selectTableIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name); - @Select( - "SELECT table_id as tableId, table_name as tableName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_id as schemaId, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND table_name = #{tableName} AND deleted_at = 0") + @SelectProvider( + type = TableMetaSQLProviderFactory.class, + method = "selectTableMetaBySchemaIdAndName") TablePO selectTableMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name); - @Select( - "SELECT table_id as tableId, table_name as tableName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_id as schemaId, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE table_id = #{tableId} AND deleted_at = 0") + @SelectProvider(type = TableMetaSQLProviderFactory.class, method = "selectTableMetaById") TablePO selectTableMetaById(@Param("tableId") Long tableId); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(table_id, table_name, metalake_id," - + " catalog_id, schema_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tableMeta.tableId}," - + " #{tableMeta.tableName}," - + " #{tableMeta.metalakeId}," - + " #{tableMeta.catalogId}," - + " #{tableMeta.schemaId}," - + " #{tableMeta.auditInfo}," - + " #{tableMeta.currentVersion}," - + " #{tableMeta.lastVersion}," - + " #{tableMeta.deletedAt}" - + " )") + @InsertProvider(type = TableMetaSQLProviderFactory.class, method = "insertTableMeta") void insertTableMeta(@Param("tableMeta") TablePO tablePO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(table_id, table_name, metalake_id," - + " catalog_id, schema_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tableMeta.tableId}," - + " #{tableMeta.tableName}," - + " #{tableMeta.metalakeId}," - + " #{tableMeta.catalogId}," - + " #{tableMeta.schemaId}," - + " #{tableMeta.auditInfo}," - + " #{tableMeta.currentVersion}," - + " #{tableMeta.lastVersion}," - + " #{tableMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " table_name = #{tableMeta.tableName}," - + " metalake_id = #{tableMeta.metalakeId}," - + " catalog_id = #{tableMeta.catalogId}," - + " schema_id = #{tableMeta.schemaId}," - + " audit_info = #{tableMeta.auditInfo}," - + " current_version = #{tableMeta.currentVersion}," - + " last_version = #{tableMeta.lastVersion}," - + " deleted_at = #{tableMeta.deletedAt}") + @InsertProvider( + type = TableMetaSQLProviderFactory.class, + method = "insertTableMetaOnDuplicateKeyUpdate") void insertTableMetaOnDuplicateKeyUpdate(@Param("tableMeta") TablePO tablePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET table_name = #{newTableMeta.tableName}," - + " metalake_id = #{newTableMeta.metalakeId}," - + " catalog_id = #{newTableMeta.catalogId}," - + " schema_id = #{newTableMeta.schemaId}," - + " audit_info = #{newTableMeta.auditInfo}," - + " current_version = #{newTableMeta.currentVersion}," - + " last_version = #{newTableMeta.lastVersion}," - + " deleted_at = #{newTableMeta.deletedAt}" - + " WHERE table_id = #{oldTableMeta.tableId}" - + " AND table_name = #{oldTableMeta.tableName}" - + " AND metalake_id = #{oldTableMeta.metalakeId}" - + " AND catalog_id = #{oldTableMeta.catalogId}" - + " AND schema_id = #{oldTableMeta.schemaId}" - + " AND audit_info = #{oldTableMeta.auditInfo}" - + " AND current_version = #{oldTableMeta.currentVersion}" - + " AND last_version = #{oldTableMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = TableMetaSQLProviderFactory.class, method = "updateTableMeta") Integer updateTableMeta( @Param("newTableMeta") TablePO newTablePO, @Param("oldTableMeta") TablePO oldTablePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE table_id = #{tableId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasByTableId") Integer softDeleteTableMetasByTableId(@Param("tableId") Long tableId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasByMetalakeId") Integer softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasByCatalogId") Integer softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasBySchemaId") Integer softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = TableMetaSQLProviderFactory.class, + method = "deleteTableMetasByLegacyTimeline") Integer deleteTableMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java new file mode 100644 index 00000000000..833ba9a059d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.TablePO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TableMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TableMetaMySQLProvider(), + JDBCBackendType.H2, new TableMetaH2Provider()); + + public static TableMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TableMetaMySQLProvider extends TableMetaBaseSQLProvider {} + + static class TableMetaH2Provider extends TableMetaBaseSQLProvider {} + + public static String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().listTablePOsBySchemaId(schemaId); + } + + public static String selectTableIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return getProvider().selectTableIdBySchemaIdAndName(schemaId, name); + } + + public static String selectTableMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return getProvider().selectTableMetaBySchemaIdAndName(schemaId, name); + } + + public static String selectTableMetaById(@Param("tableId") Long tableId) { + return getProvider().selectTableMetaById(tableId); + } + + public static String insertTableMeta(@Param("tableMeta") TablePO tablePO) { + return getProvider().insertTableMeta(tablePO); + } + + public static String insertTableMetaOnDuplicateKeyUpdate(@Param("tableMeta") TablePO tablePO) { + return getProvider().insertTableMetaOnDuplicateKeyUpdate(tablePO); + } + + public static String updateTableMeta( + @Param("newTableMeta") TablePO newTablePO, @Param("oldTableMeta") TablePO oldTablePO) { + return getProvider().updateTableMeta(newTablePO, oldTablePO); + } + + public static String softDeleteTableMetasByTableId(@Param("tableId") Long tableId) { + return getProvider().softDeleteTableMetasByTableId(tableId); + } + + public static String softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTableMetasByMetalakeId(metalakeId); + } + + public static String softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteTableMetasByCatalogId(catalogId); + } + + public static String softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteTableMetasBySchemaId(schemaId); + } + + public static String deleteTableMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTableMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaBaseSQLProvider.java new file mode 100644 index 00000000000..aa46bbd7f9c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaBaseSQLProvider.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.TopicPO; +import org.apache.ibatis.annotations.Param; + +public class TopicMetaBaseSQLProvider { + + public String insertTopicMeta(@Param("topicMeta") TopicPO topicPO) { + return "INSERT INTO " + + TABLE_NAME + + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," + + " comment, properties, audit_info, current_version, last_version," + + " deleted_at)" + + " VALUES(" + + " #{topicMeta.topicId}," + + " #{topicMeta.topicName}," + + " #{topicMeta.metalakeId}," + + " #{topicMeta.catalogId}," + + " #{topicMeta.schemaId}," + + " #{topicMeta.comment}," + + " #{topicMeta.properties}," + + " #{topicMeta.auditInfo}," + + " #{topicMeta.currentVersion}," + + " #{topicMeta.lastVersion}," + + " #{topicMeta.deletedAt}" + + " )"; + } + + public String insertTopicMetaOnDuplicateKeyUpdate(@Param("topicMeta") TopicPO topicPO) { + return "INSERT INTO " + + TABLE_NAME + + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," + + " comment, properties, audit_info, current_version, last_version," + + " deleted_at)" + + " VALUES(" + + " #{topicMeta.topicId}," + + " #{topicMeta.topicName}," + + " #{topicMeta.metalakeId}," + + " #{topicMeta.catalogId}," + + " #{topicMeta.schemaId}," + + " #{topicMeta.comment}," + + " #{topicMeta.properties}," + + " #{topicMeta.auditInfo}," + + " #{topicMeta.currentVersion}," + + " #{topicMeta.lastVersion}," + + " #{topicMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " topic_name = #{topicMeta.topicName}," + + " metalake_id = #{topicMeta.metalakeId}," + + " catalog_id = #{topicMeta.catalogId}," + + " schema_id = #{topicMeta.schemaId}," + + " comment = #{topicMeta.comment}," + + " properties = #{topicMeta.properties}," + + " audit_info = #{topicMeta.auditInfo}," + + " current_version = #{topicMeta.currentVersion}," + + " last_version = #{topicMeta.lastVersion}," + + " deleted_at = #{topicMeta.deletedAt}"; + } + + public String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) { + return "SELECT topic_id as topicId, topic_name as topicName, metalake_id as metalakeId," + + " catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String selectTopicMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String topicName) { + return "SELECT topic_id as topicId, topic_name as topicName," + + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName} AND deleted_at = 0"; + } + + public String selectTopicMetaById(@Param("topicId") Long topicId) { + return "SELECT topic_id as topicId, topic_name as topicName," + + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE topic_id = #{topicId} AND deleted_at = 0"; + } + + public String updateTopicMeta( + @Param("newTopicMeta") TopicPO newTopicPO, @Param("oldTopicMeta") TopicPO oldTopicPO) { + return "UPDATE " + + TABLE_NAME + + " SET topic_name = #{newTopicMeta.topicName}," + + " metalake_id = #{newTopicMeta.metalakeId}," + + " catalog_id = #{newTopicMeta.catalogId}," + + " schema_id = #{newTopicMeta.schemaId}," + + " comment = #{newTopicMeta.comment}," + + " properties = #{newTopicMeta.properties}," + + " audit_info = #{newTopicMeta.auditInfo}," + + " current_version = #{newTopicMeta.currentVersion}," + + " last_version = #{newTopicMeta.lastVersion}," + + " deleted_at = #{newTopicMeta.deletedAt}" + + " WHERE topic_id = #{oldTopicMeta.topicId}" + + " AND topic_name = #{oldTopicMeta.topicName}" + + " AND metalake_id = #{oldTopicMeta.metalakeId}" + + " AND catalog_id = #{oldTopicMeta.catalogId}" + + " AND schema_id = #{oldTopicMeta.schemaId}" + + " AND comment = #{oldTopicMeta.comment}" + + " AND properties = #{oldTopicMeta.properties}" + + " AND audit_info = #{oldTopicMeta.auditInfo}" + + " AND current_version = #{oldTopicMeta.currentVersion}" + + " AND last_version = #{oldTopicMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String selectTopicIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String name) { + return "SELECT topic_id as topicId FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName}" + + " AND deleted_at = 0"; + } + + public String softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE topic_id = #{topicId} AND deleted_at = 0"; + } + + public String softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String deleteTopicMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java index d8d08286e38..8c194caff4f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java @@ -20,173 +20,68 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TopicPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; public interface TopicMetaMapper { String TABLE_NAME = "topic_meta"; - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," - + " comment, properties, audit_info, current_version, last_version," - + " deleted_at)" - + " VALUES(" - + " #{topicMeta.topicId}," - + " #{topicMeta.topicName}," - + " #{topicMeta.metalakeId}," - + " #{topicMeta.catalogId}," - + " #{topicMeta.schemaId}," - + " #{topicMeta.comment}," - + " #{topicMeta.properties}," - + " #{topicMeta.auditInfo}," - + " #{topicMeta.currentVersion}," - + " #{topicMeta.lastVersion}," - + " #{topicMeta.deletedAt}" - + " )") + @InsertProvider(type = TopicMetaSQLProviderFactory.class, method = "insertTopicMeta") void insertTopicMeta(@Param("topicMeta") TopicPO topicPO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," - + " comment, properties, audit_info, current_version, last_version," - + " deleted_at)" - + " VALUES(" - + " #{topicMeta.topicId}," - + " #{topicMeta.topicName}," - + " #{topicMeta.metalakeId}," - + " #{topicMeta.catalogId}," - + " #{topicMeta.schemaId}," - + " #{topicMeta.comment}," - + " #{topicMeta.properties}," - + " #{topicMeta.auditInfo}," - + " #{topicMeta.currentVersion}," - + " #{topicMeta.lastVersion}," - + " #{topicMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " topic_name = #{topicMeta.topicName}," - + " metalake_id = #{topicMeta.metalakeId}," - + " catalog_id = #{topicMeta.catalogId}," - + " schema_id = #{topicMeta.schemaId}," - + " comment = #{topicMeta.comment}," - + " properties = #{topicMeta.properties}," - + " audit_info = #{topicMeta.auditInfo}," - + " current_version = #{topicMeta.currentVersion}," - + " last_version = #{topicMeta.lastVersion}," - + " deleted_at = #{topicMeta.deletedAt}") + @InsertProvider( + type = TopicMetaSQLProviderFactory.class, + method = "insertTopicMetaOnDuplicateKeyUpdate") void insertTopicMetaOnDuplicateKeyUpdate(@Param("topicMeta") TopicPO topicPO); - @Select( - "SELECT topic_id as topicId, topic_name as topicName, metalake_id as metalakeId," - + " catalog_id as catalogId, schema_id as schemaId," - + " comment as comment, properties as properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsBySchemaId") List listTopicPOsBySchemaId(@Param("schemaId") Long schemaId); - @Select( - "SELECT topic_id as topicId, topic_name as topicName," - + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," - + " comment as comment, properties as properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName} AND deleted_at = 0") + @SelectProvider( + type = TopicMetaSQLProviderFactory.class, + method = "selectTopicMetaBySchemaIdAndName") TopicPO selectTopicMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String topicName); - @Select( - "SELECT topic_id as topicId, topic_name as topicName," - + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," - + " comment as comment, properties as properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE topic_id = #{topicId} AND deleted_at = 0") + @SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "selectTopicMetaById") TopicPO selectTopicMetaById(@Param("topicId") Long topicId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET topic_name = #{newTopicMeta.topicName}," - + " metalake_id = #{newTopicMeta.metalakeId}," - + " catalog_id = #{newTopicMeta.catalogId}," - + " schema_id = #{newTopicMeta.schemaId}," - + " comment = #{newTopicMeta.comment}," - + " properties = #{newTopicMeta.properties}," - + " audit_info = #{newTopicMeta.auditInfo}," - + " current_version = #{newTopicMeta.currentVersion}," - + " last_version = #{newTopicMeta.lastVersion}," - + " deleted_at = #{newTopicMeta.deletedAt}" - + " WHERE topic_id = #{oldTopicMeta.topicId}" - + " AND topic_name = #{oldTopicMeta.topicName}" - + " AND metalake_id = #{oldTopicMeta.metalakeId}" - + " AND catalog_id = #{oldTopicMeta.catalogId}" - + " AND schema_id = #{oldTopicMeta.schemaId}" - + " AND comment = #{oldTopicMeta.comment}" - + " AND properties = #{oldTopicMeta.properties}" - + " AND audit_info = #{oldTopicMeta.auditInfo}" - + " AND current_version = #{oldTopicMeta.currentVersion}" - + " AND last_version = #{oldTopicMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = TopicMetaSQLProviderFactory.class, method = "updateTopicMeta") Integer updateTopicMeta( @Param("newTopicMeta") TopicPO newTopicPO, @Param("oldTopicMeta") TopicPO oldTopicPO); - @Select( - "SELECT topic_id as topicId FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName}" - + " AND deleted_at = 0") + @SelectProvider( + type = TopicMetaSQLProviderFactory.class, + method = "selectTopicIdBySchemaIdAndName") Long selectTopicIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String name); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE topic_id = #{topicId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasByTopicId") Integer softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasByCatalogId") Integer softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasByMetalakeId") Integer softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasBySchemaId") Integer softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = TopicMetaSQLProviderFactory.class, + method = "deleteTopicMetasByLegacyTimeline") Integer deleteTopicMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java new file mode 100644 index 00000000000..9a417e011a7 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.TopicPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TopicMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TopicMetaMySQLProvider(), + JDBCBackendType.H2, new TopicMetaH2Provider()); + + public static TopicMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TopicMetaMySQLProvider extends TopicMetaBaseSQLProvider {} + + static class TopicMetaH2Provider extends TopicMetaBaseSQLProvider {} + + public static String insertTopicMeta(@Param("topicMeta") TopicPO topicPO) { + return getProvider().insertTopicMeta(topicPO); + } + + public static String insertTopicMetaOnDuplicateKeyUpdate(@Param("topicMeta") TopicPO topicPO) { + return getProvider().insertTopicMetaOnDuplicateKeyUpdate(topicPO); + } + + public static String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().listTopicPOsBySchemaId(schemaId); + } + + public static String selectTopicMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String topicName) { + return getProvider().selectTopicMetaBySchemaIdAndName(schemaId, topicName); + } + + public static String selectTopicMetaById(@Param("topicId") Long topicId) { + return getProvider().selectTopicMetaById(topicId); + } + + public static String updateTopicMeta( + @Param("newTopicMeta") TopicPO newTopicPO, @Param("oldTopicMeta") TopicPO oldTopicPO) { + return getProvider().updateTopicMeta(newTopicPO, oldTopicPO); + } + + public static String selectTopicIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String name) { + return getProvider().selectTopicIdBySchemaIdAndName(schemaId, name); + } + + public static String softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId) { + return getProvider().softDeleteTopicMetasByTopicId(topicId); + } + + public static String softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteTopicMetasByCatalogId(catalogId); + } + + public static String softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTopicMetasByMetalakeId(metalakeId); + } + + public static String softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteTopicMetasBySchemaId(schemaId); + } + + public static String deleteTopicMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTopicMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index 684252f17ee..9d928271623 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -26,6 +26,7 @@ import org.apache.commons.pool2.impl.BaseObjectPoolConfig; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper; @@ -73,7 +74,9 @@ private SqlSessionFactoryHelper() {} public void init(Config config) { // Initialize the data source BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL)); + String jdbcUrl = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL); + JDBCBackendType jdbcType = JDBCBackendType.fromURI(jdbcUrl); + dataSource.setUrl(jdbcUrl); dataSource.setDriverClassName(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)); dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER)); dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)); @@ -102,6 +105,7 @@ public void init(Config config) { // Initialize the configuration Configuration configuration = new Configuration(environment); + configuration.setDatabaseId(jdbcType.name().toLowerCase()); configuration.addMapper(MetalakeMetaMapper.class); configuration.addMapper(CatalogMetaMapper.class); configuration.addMapper(SchemaMetaMapper.class); From a0cb17450ce90bd6fe8684563c891fc6bb6c4c43 Mon Sep 17 00:00:00 2001 From: FANNG Date: Fri, 6 Sep 2024 00:28:20 +0800 Subject: [PATCH 2/8] [#4758] correct Iceberg document location is immutable not reserved (#4808) ### What changes were proposed in this pull request? correct Iceberg document location is immutable not reserved ### Why are the changes needed? Fix: #4758 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? document --- docs/lakehouse-iceberg-catalog.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 738f0e432bb..0fc3af157c0 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -284,16 +284,21 @@ You can pass [Iceberg table properties](https://iceberg.apache.org/docs/1.5.2/co The Gravitino server doesn't allow passing the following reserved fields. -| Configuration item | Description | -|---------------------------------|---------------------------------------------------------| -| `comment` | The table comment. | -| `creator` | The table creator. | -| `location` | Iceberg location for table storage. | -| `current-snapshot-id` | The snapshot represents the current state of the table. | -| `cherry-pick-snapshot-id` | Selecting a specific snapshot in a merge operation. | -| `sort-order` | Selecting a specific snapshot in a merge operation. | -| `identifier-fields` | The identifier fields for defining the table. | -| `write.distribution-mode` | Defines distribution of write data | +| Configuration item | Description | Since Version | +|---------------------------|--------------------------------------------------------------------------------------|---------------| +| `comment` | The table comment, please use `comment` field in table meta instead. | 0.2.0 | +| `creator` | The table creator. | 0.2.0 | +| `current-snapshot-id` | The snapshot represents the current state of the table. | 0.2.0 | +| `cherry-pick-snapshot-id` | Selecting a specific snapshot in a merge operation. | 0.2.0 | +| `sort-order` | Iceberg table sort order, please use `SortOrder` in table meta instead. | 0.2.0 | +| `identifier-fields` | The identifier fields for defining the table. | 0.2.0 | +| `write.distribution-mode` | Defines distribution of write data, please use `distribution` in table meta instead. | 0.2.0 | + +Gravitino server doesn't allow to change such properties: + +| Configuration item | Description | Default value | Required | Since Version | +|--------------------|----------------------------------------------|---------------|----------|---------------| +| `location` | Iceberg location for table storage. | None | No | 0.2.0 | ### Table indexes From 0403f3166f0882d663e8db3c126f93b1a6522701 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 6 Sep 2024 13:48:40 +0800 Subject: [PATCH 3/8] [#4868] improvement(core): Modify tag and user related mapper class to use the new framework to support multiple JDBC backend. (#4869) ### What changes were proposed in this pull request? Change tag, user, and role-related code to support the new framework introduced by #4832 ### Why are the changes needed? To support multiple JDBC backend. Fix: #4868 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Existing UTs and ITs. --- .../mapper/GroupMetaBaseSQLProvider.java | 146 +++++++++++++ .../relational/mapper/GroupMetaMapper.java | 123 ++--------- .../mapper/GroupMetaSQLProviderFactory.java | 89 ++++++++ .../mapper/GroupRoleRelBaseSQLProvider.java | 125 +++++++++++ .../relational/mapper/GroupRoleRelMapper.java | 105 ++-------- .../GroupRoleRelSQLProviderFactory.java | 83 ++++++++ .../mapper/OwnerMetaBaseSQLProvider.java | 181 ++++++++++++++++ .../relational/mapper/OwnerMetaMapper.java | 161 +++----------- .../mapper/OwnerMetaSQLProviderFactory.java | 97 +++++++++ .../mapper/RoleMetaBaseSQLProvider.java | 160 ++++++++++++++ .../relational/mapper/RoleMetaMapper.java | 136 +++--------- .../mapper/RoleMetaSQLProviderFactory.java | 93 +++++++++ .../SecurableObjectBaseSQLProvider.java | 86 ++++++++ .../mapper/SecurableObjectMapper.java | 69 ++---- .../SecurableObjectSQLProviderFactory.java | 73 +++++++ .../mapper/TagMetaBaseSQLProvider.java | 197 ++++++++++++++++++ .../relational/mapper/TagMetaMapper.java | 167 ++------------- .../mapper/TagMetaSQLProviderFactory.java | 96 +++++++++ .../TagMetadataObjectRelBaseSQLProvider.java | 154 ++++++++++++++ .../mapper/TagMetadataObjectRelMapper.java | 132 +++--------- ...agMetadataObjectRelSQLProviderFactory.java | 99 +++++++++ .../mapper/UserMetaBaseSQLProvider.java | 147 +++++++++++++ .../relational/mapper/UserMetaMapper.java | 123 ++--------- .../mapper/UserMetaSQLProviderFactory.java | 91 ++++++++ .../mapper/UserRoleRelBaseSQLProvider.java | 126 +++++++++++ .../relational/mapper/UserRoleRelMapper.java | 108 ++-------- .../mapper/UserRoleRelSQLProviderFactory.java | 84 ++++++++ .../relational/service/SchemaMetaService.java | 3 +- 28 files changed, 2339 insertions(+), 915 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelBaseSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaBaseSQLProvider.java new file mode 100644 index 00000000000..686bc42c565 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaBaseSQLProvider.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.GroupMetaMapper.GROUP_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.GROUP_ROLE_RELATION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.GroupPO; +import org.apache.ibatis.annotations.Param; + +public class GroupMetaBaseSQLProvider { + + public String selectGroupIdBySchemaIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return "SELECT group_id as groupId FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" + + " AND deleted_at = 0"; + } + + public String selectGroupMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return "SELECT group_id as groupId, group_name as groupName," + + " metalake_id as metalakeId," + + " audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" + + " AND deleted_at = 0"; + } + + public String insertGroupMeta(@Param("groupMeta") GroupPO groupPO) { + return "INSERT INTO " + + GROUP_TABLE_NAME + + "(group_id, group_name," + + " metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )"; + } + + public String insertGroupMetaOnDuplicateKeyUpdate(@Param("groupMeta") GroupPO groupPO) { + return "INSERT INTO " + + GROUP_TABLE_NAME + + "(group_id, group_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " group_name = #{groupMeta.groupName}," + + " metalake_id = #{groupMeta.metalakeId}," + + " audit_info = #{groupMeta.auditInfo}," + + " current_version = #{groupMeta.currentVersion}," + + " last_version = #{groupMeta.lastVersion}," + + " deleted_at = #{groupMeta.deletedAt}"; + } + + public String softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + public String softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String updateGroupMeta( + @Param("newGroupMeta") GroupPO newGroupPO, @Param("oldGroupMeta") GroupPO oldGroupPO) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET group_name = #{newGroupMeta.groupName}," + + " metalake_id = #{newGroupMeta.metalakeId}," + + " audit_info = #{newGroupMeta.auditInfo}," + + " current_version = #{newGroupMeta.currentVersion}," + + " last_version = #{newGroupMeta.lastVersion}," + + " deleted_at = #{newGroupMeta.deletedAt}" + + " WHERE group_id = #{oldGroupMeta.groupId}" + + " AND group_name = #{oldGroupMeta.groupName}" + + " AND metalake_id = #{oldGroupMeta.metalakeId}" + + " AND audit_info = #{oldGroupMeta.auditInfo}" + + " AND current_version = #{oldGroupMeta.currentVersion}" + + " AND last_version = #{oldGroupMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String listGroupsByRoleId(@Param("roleId") Long roleId) { + return "SELECT gr.group_id as groupId, gr.group_name as groupName," + + " gr.metalake_id as metalakeId," + + " gr.audit_info as auditInfo, gr.current_version as currentVersion," + + " gr.last_version as lastVersion, gr.deleted_at as deletedAt" + + " FROM " + + GROUP_TABLE_NAME + + " gr JOIN " + + GROUP_ROLE_RELATION_TABLE_NAME + + " re ON gr.group_id = re.group_id" + + " WHERE re.role_id = #{roleId}" + + " AND gr.deleted_at = 0 AND re.deleted_at = 0"; + } + + public String deleteGroupMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + GROUP_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java index 6250c2a9a6c..5743095dd72 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.GroupPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -39,121 +39,44 @@ public interface GroupMetaMapper { String GROUP_TABLE_NAME = "group_meta"; String GROUP_ROLE_RELATION_TABLE_NAME = "group_role_rel"; - @Select( - "SELECT group_id as groupId FROM " - + GROUP_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" - + " AND deleted_at = 0") + @SelectProvider( + type = GroupMetaSQLProviderFactory.class, + method = "selectGroupIdBySchemaIdAndName") Long selectGroupIdBySchemaIdAndName( @Param("metalakeId") Long metalakeId, @Param("groupName") String name); - @Select( - "SELECT group_id as groupId, group_name as groupName," - + " metalake_id as metalakeId," - + " audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + GROUP_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" - + " AND deleted_at = 0") + @SelectProvider( + type = GroupMetaSQLProviderFactory.class, + method = "selectGroupMetaByMetalakeIdAndName") GroupPO selectGroupMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("groupName") String name); - @Insert( - "INSERT INTO " - + GROUP_TABLE_NAME - + "(group_id, group_name," - + " metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{groupMeta.groupId}," - + " #{groupMeta.groupName}," - + " #{groupMeta.metalakeId}," - + " #{groupMeta.auditInfo}," - + " #{groupMeta.currentVersion}," - + " #{groupMeta.lastVersion}," - + " #{groupMeta.deletedAt}" - + " )") + @InsertProvider(type = GroupMetaSQLProviderFactory.class, method = "insertGroupMeta") void insertGroupMeta(@Param("groupMeta") GroupPO groupPO); - @Insert( - "INSERT INTO " - + GROUP_TABLE_NAME - + "(group_id, group_name," - + "metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{groupMeta.groupId}," - + " #{groupMeta.groupName}," - + " #{groupMeta.metalakeId}," - + " #{groupMeta.auditInfo}," - + " #{groupMeta.currentVersion}," - + " #{groupMeta.lastVersion}," - + " #{groupMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " group_name = #{groupMeta.groupName}," - + " metalake_id = #{groupMeta.metalakeId}," - + " audit_info = #{groupMeta.auditInfo}," - + " current_version = #{groupMeta.currentVersion}," - + " last_version = #{groupMeta.lastVersion}," - + " deleted_at = #{groupMeta.deletedAt}") + @InsertProvider( + type = GroupMetaSQLProviderFactory.class, + method = "insertGroupMetaOnDuplicateKeyUpdate") void insertGroupMetaOnDuplicateKeyUpdate(@Param("groupMeta") GroupPO groupPO); - @Update( - "UPDATE " - + GROUP_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE group_id = #{groupId} AND deleted_at = 0") + @UpdateProvider(type = GroupMetaSQLProviderFactory.class, method = "softDeleteGroupMetaByGroupId") void softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId); - @Update( - "UPDATE " - + GROUP_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = GroupMetaSQLProviderFactory.class, + method = "softDeleteGroupMetasByMetalakeId") void softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + GROUP_TABLE_NAME - + " SET group_name = #{newGroupMeta.groupName}," - + " metalake_id = #{newGroupMeta.metalakeId}," - + " audit_info = #{newGroupMeta.auditInfo}," - + " current_version = #{newGroupMeta.currentVersion}," - + " last_version = #{newGroupMeta.lastVersion}," - + " deleted_at = #{newGroupMeta.deletedAt}" - + " WHERE group_id = #{oldGroupMeta.groupId}" - + " AND group_name = #{oldGroupMeta.groupName}" - + " AND metalake_id = #{oldGroupMeta.metalakeId}" - + " AND audit_info = #{oldGroupMeta.auditInfo}" - + " AND current_version = #{oldGroupMeta.currentVersion}" - + " AND last_version = #{oldGroupMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = GroupMetaSQLProviderFactory.class, method = "updateGroupMeta") Integer updateGroupMeta( @Param("newGroupMeta") GroupPO newGroupPO, @Param("oldGroupMeta") GroupPO oldGroupPO); - @Select( - "SELECT gr.group_id as groupId, gr.group_name as groupName," - + " gr.metalake_id as metalakeId," - + " gr.audit_info as auditInfo, gr.current_version as currentVersion," - + " gr.last_version as lastVersion, gr.deleted_at as deletedAt" - + " FROM " - + GROUP_TABLE_NAME - + " gr JOIN " - + GROUP_ROLE_RELATION_TABLE_NAME - + " re ON gr.group_id = re.group_id" - + " WHERE re.role_id = #{roleId}" - + " AND gr.deleted_at = 0 AND re.deleted_at = 0") + @SelectProvider(type = GroupMetaSQLProviderFactory.class, method = "listGroupsByRoleId") List listGroupsByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + GROUP_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = GroupMetaSQLProviderFactory.class, + method = "deleteGroupMetasByLegacyTimeline") Integer deleteGroupMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java new file mode 100644 index 00000000000..59e45a6dc6b --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.GroupPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class GroupMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new GroupMetaMySQLProvider(), + JDBCBackendType.H2, new GroupMetaH2Provider()); + + public static GroupMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class GroupMetaMySQLProvider extends GroupMetaBaseSQLProvider {} + + static class GroupMetaH2Provider extends GroupMetaBaseSQLProvider {} + + public static String selectGroupIdBySchemaIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return getProvider().selectGroupIdBySchemaIdAndName(metalakeId, name); + } + + public static String selectGroupMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return getProvider().selectGroupMetaByMetalakeIdAndName(metalakeId, name); + } + + public static String insertGroupMeta(@Param("groupMeta") GroupPO groupPO) { + return getProvider().insertGroupMeta(groupPO); + } + + public static String insertGroupMetaOnDuplicateKeyUpdate(@Param("groupMeta") GroupPO groupPO) { + return getProvider().insertGroupMetaOnDuplicateKeyUpdate(groupPO); + } + + public static String softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId) { + return getProvider().softDeleteGroupMetaByGroupId(groupId); + } + + public static String softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteGroupMetasByMetalakeId(metalakeId); + } + + public static String updateGroupMeta( + @Param("newGroupMeta") GroupPO newGroupPO, @Param("oldGroupMeta") GroupPO oldGroupPO) { + return getProvider().updateGroupMeta(newGroupPO, oldGroupPO); + } + + public static String listGroupsByRoleId(@Param("roleId") Long roleId) { + return getProvider().listGroupsByRoleId(roleId); + } + + public static String deleteGroupMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteGroupMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelBaseSQLProvider.java new file mode 100644 index 00000000000..ea883681570 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelBaseSQLProvider.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; +import org.apache.ibatis.annotations.Param; + +public class GroupRoleRelBaseSQLProvider { + public String batchInsertGroupRoleRel( + @Param("groupRoleRels") List groupRoleRelPOS) { + return ""; + } + + public String batchInsertGroupRoleRelOnDuplicateKeyUpdate( + @Param("groupRoleRels") List groupRoleRelPOS) { + return ""; + } + + public String softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + public String softDeleteGroupRoleRelByGroupAndRoles( + @Param("groupId") Long groupId, @Param("roleIds") List roleIds) { + return ""; + } + + public String softDeleteGroupRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE group_id IN (SELECT group_id FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + public String softDeleteGroupRoleRelByRoleId(Long roleId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String deleteGroupRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + GROUP_ROLE_RELATION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java index c428341d7a4..a40ee8daae9 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java @@ -21,10 +21,9 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -38,101 +37,39 @@ public interface GroupRoleRelMapper { String GROUP_TABLE_NAME = "group_meta"; String GROUP_ROLE_RELATION_TABLE_NAME = "group_role_rel"; - @Insert({ - "" - }) + @InsertProvider(type = GroupRoleRelSQLProviderFactory.class, method = "batchInsertGroupRoleRel") void batchInsertGroupRoleRel(@Param("groupRoleRels") List groupRoleRelPOS); - @Insert({ - "" - }) + @InsertProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "batchInsertGroupRoleRelOnDuplicateKeyUpdate") void batchInsertGroupRoleRelOnDuplicateKeyUpdate( @Param("groupRoleRels") List groupRoleRelPOS); - @Update( - "UPDATE " - + GROUP_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE group_id = #{groupId} AND deleted_at = 0") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByGroupId") void softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId); - @Update({ - "" - }) + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByGroupAndRoles") void softDeleteGroupRoleRelByGroupAndRoles( @Param("groupId") Long groupId, @Param("roleIds") List roleIds); - @Update( - "UPDATE " - + GROUP_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE group_id IN (SELECT group_id FROM " - + GROUP_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" - + " AND deleted_at = 0") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByMetalakeId") void softDeleteGroupRoleRelByMetalakeId(Long metalakeId); - @Update( - "UPDATE " - + GROUP_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByRoleId") void softDeleteGroupRoleRelByRoleId(Long roleId); - @Delete( - "DELETE FROM " - + GROUP_ROLE_RELATION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "deleteGroupRoleRelMetasByLegacyTimeline") Integer deleteGroupRoleRelMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java new file mode 100644 index 00000000000..4fd047abeb1 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class GroupRoleRelSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new GroupRoleRelMySQLProvider(), + JDBCBackendType.H2, new GroupRoleRelH2Provider()); + + public static GroupRoleRelBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class GroupRoleRelMySQLProvider extends GroupRoleRelBaseSQLProvider {} + + static class GroupRoleRelH2Provider extends GroupRoleRelBaseSQLProvider {} + + public static String batchInsertGroupRoleRel( + @Param("groupRoleRels") List groupRoleRelPOS) { + return getProvider().batchInsertGroupRoleRel(groupRoleRelPOS); + } + + public static String batchInsertGroupRoleRelOnDuplicateKeyUpdate( + @Param("groupRoleRels") List groupRoleRelPOS) { + return getProvider().batchInsertGroupRoleRelOnDuplicateKeyUpdate(groupRoleRelPOS); + } + + public static String softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId) { + return getProvider().softDeleteGroupRoleRelByGroupId(groupId); + } + + public static String softDeleteGroupRoleRelByGroupAndRoles( + @Param("groupId") Long groupId, @Param("roleIds") List roleIds) { + return getProvider().softDeleteGroupRoleRelByGroupAndRoles(groupId, roleIds); + } + + public static String softDeleteGroupRoleRelByMetalakeId(Long metalakeId) { + return getProvider().softDeleteGroupRoleRelByMetalakeId(metalakeId); + } + + public static String softDeleteGroupRoleRelByRoleId(Long roleId) { + return getProvider().softDeleteGroupRoleRelByRoleId(roleId); + } + + public static String deleteGroupRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteGroupRoleRelMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaBaseSQLProvider.java new file mode 100644 index 00000000000..0d46034a5db --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaBaseSQLProvider.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper.OWNER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.OwnerRelPO; +import org.apache.ibatis.annotations.Param; + +public class OwnerMetaBaseSQLProvider { + public String selectUserOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT ut.user_id as userId," + + " ut.user_name as userName," + + " ut.metalake_id as metalakeId," + + " ut.audit_info as auditInfo," + + " ut.current_version as currentVersion," + + " ut.last_version as lastVersion," + + " ut.deleted_at as deletedAt" + + " FROM " + + OWNER_TABLE_NAME + + " ot JOIN " + + UserMetaMapper.USER_TABLE_NAME + + " ut ON ut.user_id = ot.owner_id" + + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" + + " ot.metadata_object_type = #{metadataObjectType} AND" + + " ot.owner_type = 'USER' AND" + + " ot.deleted_at = 0 AND ut.deleted_at = 0"; + } + + public String selectGroupOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT gt.group_id as groupId," + + " gt.group_name as groupName," + + " gt.metalake_id as metalakeId," + + " gt.audit_info as auditInfo," + + " gt.current_version as currentVersion," + + " gt.last_version as lastVersion," + + " gt.deleted_at as deletedAt" + + " FROM " + + OWNER_TABLE_NAME + + " ot JOIN " + + GroupMetaMapper.GROUP_TABLE_NAME + + " gt ON gt.group_id = ot.owner_id" + + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" + + " ot.metadata_object_type = #{metadataObjectType} AND" + + " ot.owner_type = 'GROUP' AND" + + " ot.deleted_at = 0 AND gt.deleted_at = 0"; + } + + public String insertOwnerRel(@Param("ownerRelPO") OwnerRelPO ownerRelPO) { + return "INSERT INTO " + + OWNER_TABLE_NAME + + "(metalake_id, metadata_object_id, metadata_object_type, owner_id, owner_type," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES (" + + " #{ownerRelPO.metalakeId}," + + " #{ownerRelPO.metadataObjectId}," + + " #{ownerRelPO.metadataObjectType}," + + " #{ownerRelPO.ownerId}," + + " #{ownerRelPO.ownerType}," + + " #{ownerRelPO.auditInfo}," + + " #{ownerRelPO.currentVersion}," + + " #{ownerRelPO.lastVersion}," + + " #{ownerRelPO.deletedAt}" + + ")"; + } + + public String softDeleteOwnerRelByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metadata_object_id = #{metadataObjectId} AND metadata_object_type = #{metadataObjectType} AND deleted_at = 0"; + } + + public String softDeleteOwnerRelByOwnerIdAndType( + @Param("ownerId") Long ownerId, @Param("ownerType") String ownerType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE owner_id = #{ownerId} AND owner_type = #{ownerType} AND deleted_at = 0"; + } + + public String softDeleteOwnerRelByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at =0"; + } + + public String softDeleteOwnerRelByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE EXISTS (" + + " SELECT ct.catalog_id FROM " + + CatalogMetaMapper.TABLE_NAME + + " ct WHERE ct.catalog_id = #{catalogId} AND ct.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'" + + " UNION " + + " SELECT st.catalog_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.catalog_id = #{catalogId} AND st.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.catalog_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.catalog_id = #{catalogId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.catalog_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.catalog_id = #{catalogId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.catalog_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.catalog_id = #{catalogId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND" + + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } + + public String sotDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE EXISTS (" + + " SELECT st.schema_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.schema_id = #{schemaId} AND st.deleted_at = 0 AND ot.deleted_at = 0 " + + "AND st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.schema_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.schema_id = #{schemaId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.schema_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.schema_id = #{schemaId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.schema_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.schema_id = #{schemaId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } + + public String deleteOwnerMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + OWNER_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java index 11c288324bf..455d78fb598 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java @@ -21,11 +21,10 @@ import org.apache.gravitino.storage.relational.po.GroupPO; import org.apache.gravitino.storage.relational.po.OwnerRelPO; import org.apache.gravitino.storage.relational.po.UserPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for owner meta operation SQLs. @@ -39,158 +38,52 @@ public interface OwnerMetaMapper { String OWNER_TABLE_NAME = "owner_meta"; - @Select( - "SELECT ut.user_id as userId," - + " ut.user_name as userName," - + " ut.metalake_id as metalakeId," - + " ut.audit_info as auditInfo," - + " ut.current_version as currentVersion," - + " ut.last_version as lastVersion," - + " ut.deleted_at as deletedAt" - + " FROM " - + OWNER_TABLE_NAME - + " ot JOIN " - + UserMetaMapper.USER_TABLE_NAME - + " ut ON ut.user_id = ot.owner_id" - + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" - + " ot.metadata_object_type = #{metadataObjectType} AND" - + " ot.owner_type = 'USER' AND" - + " ot.deleted_at = 0 AND ut.deleted_at = 0") + @SelectProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "selectUserOwnerMetaByMetadataObjectIdAndType") UserPO selectUserOwnerMetaByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Select( - "SELECT gt.group_id as groupId," - + " gt.group_name as groupName," - + " gt.metalake_id as metalakeId," - + " gt.audit_info as auditInfo," - + " gt.current_version as currentVersion," - + " gt.last_version as lastVersion," - + " gt.deleted_at as deletedAt" - + " FROM " - + OWNER_TABLE_NAME - + " ot JOIN " - + GroupMetaMapper.GROUP_TABLE_NAME - + " gt ON gt.group_id = ot.owner_id" - + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" - + " ot.metadata_object_type = #{metadataObjectType} AND" - + " ot.owner_type = 'GROUP' AND" - + " ot.deleted_at = 0 AND gt.deleted_at = 0") + @SelectProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "selectGroupOwnerMetaByMetadataObjectIdAndType") GroupPO selectGroupOwnerMetaByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Insert( - "INSERT INTO " - + OWNER_TABLE_NAME - + "(metalake_id, metadata_object_id, metadata_object_type, owner_id, owner_type," - + " audit_info, current_version, last_version, deleted_at)" - + " VALUES (" - + " #{ownerRelPO.metalakeId}," - + " #{ownerRelPO.metadataObjectId}," - + " #{ownerRelPO.metadataObjectType}," - + " #{ownerRelPO.ownerId}," - + " #{ownerRelPO.ownerType}," - + " #{ownerRelPO.auditInfo}," - + " #{ownerRelPO.currentVersion}," - + " #{ownerRelPO.lastVersion}," - + " #{ownerRelPO.deletedAt}" - + ")") + @InsertProvider(type = OwnerMetaSQLProviderFactory.class, method = "insertOwnerRel") void insertOwnerRel(@Param("ownerRelPO") OwnerRelPO ownerRelPO); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metadata_object_id = #{metadataObjectId} AND metadata_object_type = #{metadataObjectType} AND deleted_at = 0") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByMetadataObjectIdAndType") void softDeleteOwnerRelByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE owner_id = #{ownerId} AND owner_type = #{ownerType} AND deleted_at = 0") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByOwnerIdAndType") void softDeleteOwnerRelByOwnerIdAndType( @Param("ownerId") Long ownerId, @Param("ownerType") String ownerType); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at =0") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByMetalakeId") void softDeleteOwnerRelByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE EXISTS (" - + " SELECT ct.catalog_id FROM " - + CatalogMetaMapper.TABLE_NAME - + " ct WHERE ct.catalog_id = #{catalogId} AND ct.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'" - + " UNION " - + " SELECT st.catalog_id FROM " - + SchemaMetaMapper.TABLE_NAME - + " st WHERE st.catalog_id = #{catalogId} AND st.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" - + " UNION " - + " SELECT tt.catalog_id FROM " - + TopicMetaMapper.TABLE_NAME - + " tt WHERE tt.catalog_id = #{catalogId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" - + " UNION " - + " SELECT tat.catalog_id FROM " - + TableMetaMapper.TABLE_NAME - + " tat WHERE tat.catalog_id = #{catalogId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" - + " UNION " - + " SELECT ft.catalog_id FROM " - + FilesetMetaMapper.META_TABLE_NAME - + " ft WHERE ft.catalog_id = #{catalogId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND" - + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" - + ")") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByCatalogId") void softDeleteOwnerRelByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE EXISTS (" - + " SELECT st.schema_id FROM " - + SchemaMetaMapper.TABLE_NAME - + " st WHERE st.schema_id = #{schemaId} AND st.deleted_at = 0 AND ot.deleted_at = 0 " - + "AND st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" - + " UNION " - + " SELECT tt.schema_id FROM " - + TopicMetaMapper.TABLE_NAME - + " tt WHERE tt.schema_id = #{schemaId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" - + " UNION " - + " SELECT tat.schema_id FROM " - + TableMetaMapper.TABLE_NAME - + " tat WHERE tat.schema_id = #{schemaId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" - + " UNION " - + " SELECT ft.schema_id FROM " - + FilesetMetaMapper.META_TABLE_NAME - + " ft WHERE ft.schema_id = #{schemaId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" - + ")") - void sotDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId); + @UpdateProvider(type = OwnerMetaSQLProviderFactory.class, method = "softDeleteOwnerRelBySchemaId") + void softDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId); - @Delete( - "DELETE FROM " - + OWNER_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "deleteOwnerMetasByLegacyTimeline") Integer deleteOwnerMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java new file mode 100644 index 00000000000..e07fd269b77 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.OwnerRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class OwnerMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new OwnerMetaMySQLProvider(), + JDBCBackendType.H2, new OwnerMetaH2Provider()); + + public static OwnerMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class OwnerMetaMySQLProvider extends OwnerMetaBaseSQLProvider {} + + static class OwnerMetaH2Provider extends OwnerMetaBaseSQLProvider {} + + public static String selectUserOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider() + .selectUserOwnerMetaByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String selectGroupOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider() + .selectGroupOwnerMetaByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String insertOwnerRel(@Param("ownerRelPO") OwnerRelPO ownerRelPO) { + return getProvider().insertOwnerRel(ownerRelPO); + } + + public static String softDeleteOwnerRelByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider() + .softDeleteOwnerRelByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String softDeleteOwnerRelByOwnerIdAndType( + @Param("ownerId") Long ownerId, @Param("ownerType") String ownerType) { + return getProvider().softDeleteOwnerRelByOwnerIdAndType(ownerId, ownerType); + } + + public static String softDeleteOwnerRelByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteOwnerRelByMetalakeId(metalakeId); + } + + public static String softDeleteOwnerRelByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteOwnerRelByCatalogId(catalogId); + } + + public static String softDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().sotDeleteOwnerRelBySchemaId(schemaId); + } + + public static String deleteOwnerMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteOwnerMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaBaseSQLProvider.java new file mode 100644 index 00000000000..dad878842a3 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaBaseSQLProvider.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.GROUP_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.ROLE_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.USER_ROLE_RELATION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.RolePO; +import org.apache.ibatis.annotations.Param; + +public class RoleMetaBaseSQLProvider { + + public String selectRoleMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String roleName) { + return "SELECT role_id as roleId, role_name as roleName," + + " metalake_id as metalakeId, properties as properties," + + " audit_info as auditInfo, current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" + + " AND deleted_at = 0"; + } + + public String selectRoleIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String name) { + return "SELECT role_id as roleId FROM " + + ROLE_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" + + " AND deleted_at = 0"; + } + + public String listRolesByUserId(@Param("userId") Long userId) { + return "SELECT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + USER_ROLE_RELATION_TABLE_NAME + + " re ON ro.role_id = re.role_id" + + " WHERE re.user_id = #{userId}" + + " AND ro.deleted_at = 0 AND re.deleted_at = 0"; + } + + public String listRolesByGroupId(Long groupId) { + return "SELECT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + GROUP_ROLE_RELATION_TABLE_NAME + + " ge ON ro.role_id = ge.role_id" + + " WHERE ge.group_id = #{groupId}" + + " AND ro.deleted_at = 0 AND ge.deleted_at = 0"; + } + + public String listRolesByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT DISTINCT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME + + " se ON ro.role_id = se.role_id" + + " WHERE se.metadata_object_id = #{metadataObjectId}" + + " AND se.type = #{metadataObjectType}" + + " AND ro.deleted_at = 0 AND se.deleted_at = 0"; + } + + public String insertRoleMeta(@Param("roleMeta") RolePO rolePO) { + return "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " )"; + } + + public String insertRoleMetaOnDuplicateKeyUpdate(@Param("roleMeta") RolePO rolePO) { + return "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " ) ON DUPLICATE KEY UPDATE" + + " role_name = #{roleMeta.roleName}," + + " metalake_id = #{roleMeta.metalakeId}," + + " properties = #{roleMeta.properties}," + + " audit_info = #{roleMeta.auditInfo}," + + " current_version = #{roleMeta.currentVersion}," + + " last_version = #{roleMeta.lastVersion}," + + " deleted_at = #{roleMeta.deletedAt}"; + } + + public String softDeleteRoleMetaByRoleId(Long roleId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteRoleMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + ROLE_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java index 2c9190d69d6..d7478c3b7f4 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.RolePO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -40,134 +40,50 @@ public interface RoleMetaMapper { String USER_ROLE_RELATION_TABLE_NAME = "user_role_rel"; String GROUP_ROLE_RELATION_TABLE_NAME = "group_role_rel"; - @Select( - "SELECT role_id as roleId, role_name as roleName," - + " metalake_id as metalakeId, properties as properties," - + " audit_info as auditInfo, current_version as currentVersion," - + " last_version as lastVersion, deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" - + " AND deleted_at = 0") + @SelectProvider( + type = RoleMetaSQLProviderFactory.class, + method = "selectRoleMetaByMetalakeIdAndName") RolePO selectRoleMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("roleName") String roleName); - @Select( - "SELECT role_id as roleId FROM " - + ROLE_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" - + " AND deleted_at = 0") + @SelectProvider( + type = RoleMetaSQLProviderFactory.class, + method = "selectRoleIdByMetalakeIdAndName") Long selectRoleIdByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("roleName") String name); - @Select( - "SELECT ro.role_id as roleId, ro.role_name as roleName," - + " ro.metalake_id as metalakeId, ro.properties as properties," - + " ro.audit_info as auditInfo, ro.current_version as currentVersion," - + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " ro JOIN " - + USER_ROLE_RELATION_TABLE_NAME - + " re ON ro.role_id = re.role_id" - + " WHERE re.user_id = #{userId}" - + " AND ro.deleted_at = 0 AND re.deleted_at = 0") + @SelectProvider(type = RoleMetaSQLProviderFactory.class, method = "listRolesByUserId") List listRolesByUserId(@Param("userId") Long userId); - @Select( - "SELECT ro.role_id as roleId, ro.role_name as roleName," - + " ro.metalake_id as metalakeId, ro.properties as properties," - + " ro.audit_info as auditInfo, ro.current_version as currentVersion," - + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " ro JOIN " - + GROUP_ROLE_RELATION_TABLE_NAME - + " ge ON ro.role_id = ge.role_id" - + " WHERE ge.group_id = #{groupId}" - + " AND ro.deleted_at = 0 AND ge.deleted_at = 0") + @SelectProvider(type = RoleMetaSQLProviderFactory.class, method = "listRolesByGroupId") List listRolesByGroupId(Long groupId); - @Select( - "SELECT DISTINCT ro.role_id as roleId, ro.role_name as roleName," - + " ro.metalake_id as metalakeId, ro.properties as properties," - + " ro.audit_info as auditInfo, ro.current_version as currentVersion," - + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " ro JOIN " - + SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME - + " se ON ro.role_id = se.role_id" - + " WHERE se.metadata_object_id = #{metadataObjectId}" - + " AND se.type = #{metadataObjectType}" - + " AND ro.deleted_at = 0 AND se.deleted_at = 0") + @SelectProvider( + type = RoleMetaSQLProviderFactory.class, + method = "listRolesByMetadataObjectIdAndType") List listRolesByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Insert( - "INSERT INTO " - + ROLE_TABLE_NAME - + "(role_id, role_name," - + " metalake_id, properties," - + " audit_info, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{roleMeta.roleId}," - + " #{roleMeta.roleName}," - + " #{roleMeta.metalakeId}," - + " #{roleMeta.properties}," - + " #{roleMeta.auditInfo}," - + " #{roleMeta.currentVersion}," - + " #{roleMeta.lastVersion}," - + " #{roleMeta.deletedAt}" - + " )") + @InsertProvider(type = RoleMetaSQLProviderFactory.class, method = "insertRoleMeta") void insertRoleMeta(@Param("roleMeta") RolePO rolePO); - @Insert( - "INSERT INTO " - + ROLE_TABLE_NAME - + "(role_id, role_name," - + " metalake_id, properties," - + " audit_info, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{roleMeta.roleId}," - + " #{roleMeta.roleName}," - + " #{roleMeta.metalakeId}," - + " #{roleMeta.properties}," - + " #{roleMeta.auditInfo}," - + " #{roleMeta.currentVersion}," - + " #{roleMeta.lastVersion}," - + " #{roleMeta.deletedAt}" - + " ) ON DUPLICATE KEY UPDATE" - + " role_name = #{roleMeta.roleName}," - + " metalake_id = #{roleMeta.metalakeId}," - + " properties = #{roleMeta.properties}," - + " audit_info = #{roleMeta.auditInfo}," - + " current_version = #{roleMeta.currentVersion}," - + " last_version = #{roleMeta.lastVersion}," - + " deleted_at = #{roleMeta.deletedAt}") + @InsertProvider( + type = RoleMetaSQLProviderFactory.class, + method = "insertRoleMetaOnDuplicateKeyUpdate") void insertRoleMetaOnDuplicateKeyUpdate(@Param("roleMeta") RolePO rolePO); - @Update( - "UPDATE " - + ROLE_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider(type = RoleMetaSQLProviderFactory.class, method = "softDeleteRoleMetaByRoleId") void softDeleteRoleMetaByRoleId(Long roleId); - @Update( - "UPDATE " - + ROLE_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = RoleMetaSQLProviderFactory.class, + method = "softDeleteRoleMetasByMetalakeId") void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + ROLE_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = RoleMetaSQLProviderFactory.class, + method = "deleteRoleMetasByLegacyTimeline") Integer deleteRoleMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java new file mode 100644 index 00000000000..bdcb45749da --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.RolePO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class RoleMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new RoleMetaMySQLProvider(), + JDBCBackendType.H2, new RoleMetaH2Provider()); + + public static RoleMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class RoleMetaMySQLProvider extends RoleMetaBaseSQLProvider {} + + static class RoleMetaH2Provider extends RoleMetaBaseSQLProvider {} + + public static String selectRoleMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String roleName) { + return getProvider().selectRoleMetaByMetalakeIdAndName(metalakeId, roleName); + } + + public static String selectRoleIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String name) { + return getProvider().selectRoleIdByMetalakeIdAndName(metalakeId, name); + } + + public static String listRolesByUserId(@Param("userId") Long userId) { + return getProvider().listRolesByUserId(userId); + } + + public static String listRolesByGroupId(Long groupId) { + return getProvider().listRolesByGroupId(groupId); + } + + public static String listRolesByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String type) { + return getProvider().listRolesByMetadataObjectIdAndType(metadataObjectId, type); + } + + public static String insertRoleMeta(@Param("roleMeta") RolePO rolePO) { + return getProvider().insertRoleMeta(rolePO); + } + + public static String insertRoleMetaOnDuplicateKeyUpdate(@Param("roleMeta") RolePO rolePO) { + return getProvider().insertRoleMetaOnDuplicateKeyUpdate(rolePO); + } + + public static String softDeleteRoleMetaByRoleId(Long roleId) { + return getProvider().softDeleteRoleMetaByRoleId(roleId); + } + + public static String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteRoleMetasByMetalakeId(metalakeId); + } + + public static String deleteRoleMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteRoleMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectBaseSQLProvider.java new file mode 100644 index 00000000000..7e0a3674642 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectBaseSQLProvider.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.ROLE_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.SecurableObjectPO; +import org.apache.ibatis.annotations.Param; + +public class SecurableObjectBaseSQLProvider { + + public String batchInsertSecurableObjects( + @Param("securableObjects") List securableObjectPOs) { + return ""; + } + + public String softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " ob SET ob.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE exists (SELECT * from " + + ROLE_TABLE_NAME + + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" + + " AND ro.deleted_at = 0) AND ob.deleted_at = 0"; + } + + public String listSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return "SELECT role_id as roleId, metadata_object_id as metadataObjectId," + + " type as type, privilege_names as privilegeNames," + + " privilege_conditions as privilegeConditions, current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String deleteSecurableObjectsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java index 9a8ae52c309..e5fa90cc914 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java @@ -20,11 +20,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.SecurableObjectPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -39,61 +39,30 @@ public interface SecurableObjectMapper { String SECURABLE_OBJECT_TABLE_NAME = "role_meta_securable_object"; String ROLE_TABLE_NAME = "role_meta"; - @Insert({ - "" - }) + @InsertProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "batchInsertSecurableObjects") void batchInsertSecurableObjects( @Param("securableObjects") List securableObjectPOs); - @Update( - "UPDATE " - + SECURABLE_OBJECT_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "softDeleteSecurableObjectsByRoleId") void softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId); - @Update( - "UPDATE " - + SECURABLE_OBJECT_TABLE_NAME - + " ob SET ob.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE exists (SELECT * from " - + ROLE_TABLE_NAME - + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" - + " AND ro.deleted_at = 0) AND ob.deleted_at = 0") + @UpdateProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "softDeleteRoleMetasByMetalakeId") void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Select( - "SELECT role_id as roleId, metadata_object_id as metadataObjectId," - + " type as type, privilege_names as privilegeNames," - + " privilege_conditions as privilegeConditions, current_version as currentVersion," - + " last_version as lastVersion, deleted_at as deletedAt" - + " FROM " - + SECURABLE_OBJECT_TABLE_NAME - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @SelectProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "listSecurableObjectsByRoleId") List listSecurableObjectsByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + SECURABLE_OBJECT_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "deleteSecurableObjectsByLegacyTimeline") Integer deleteSecurableObjectsByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java new file mode 100644 index 00000000000..6508d7db39c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.SecurableObjectPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class SecurableObjectSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new SecurableObjectMySQLProvider(), + JDBCBackendType.H2, new SecurableObjectH2Provider()); + + public static SecurableObjectBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class SecurableObjectMySQLProvider extends SecurableObjectBaseSQLProvider {} + + static class SecurableObjectH2Provider extends SecurableObjectBaseSQLProvider {} + + public static String batchInsertSecurableObjects( + @Param("securableObjects") List securableObjectPOs) { + return getProvider().batchInsertSecurableObjects(securableObjectPOs); + } + + public static String softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return getProvider().softDeleteSecurableObjectsByRoleId(roleId); + } + + public static String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteRoleMetasByMetalakeId(metalakeId); + } + + public static String listSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return getProvider().listSecurableObjectsByRoleId(roleId); + } + + public static String deleteSecurableObjectsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteSecurableObjectsByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaBaseSQLProvider.java new file mode 100644 index 00000000000..4c9ee54bfff --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaBaseSQLProvider.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TagMetaMapper.TAG_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.TagPO; +import org.apache.ibatis.annotations.Param; + +public class TagMetaBaseSQLProvider { + + public String listTagPOsByMetalake(@Param("metalakeName") String metalakeName) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId," + + " tm.tag_comment as comment," + + " tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String listTagPOsByMetalakeAndTagNames( + @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames) { + return ""; + } + + public String selectTagIdByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "SELECT tm.tag_id as tagId FROM " + + TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String selectTagMetaByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId," + + " tm.tag_comment as comment," + + " tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String insertTagMeta(@Param("tagMeta") TagPO tagPO) { + return "INSERT INTO " + + TAG_TABLE_NAME + + " (tag_id, tag_name," + + " metalake_id, tag_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tagMeta.tagId}," + + " #{tagMeta.tagName}," + + " #{tagMeta.metalakeId}," + + " #{tagMeta.comment}," + + " #{tagMeta.properties}," + + " #{tagMeta.auditInfo}," + + " #{tagMeta.currentVersion}," + + " #{tagMeta.lastVersion}," + + " #{tagMeta.deletedAt}" + + " )"; + } + + public String insertTagMetaOnDuplicateKeyUpdate(@Param("tagMeta") TagPO tagPO) { + return "INSERT INTO " + + TAG_TABLE_NAME + + "(tag_id, tag_name," + + " metalake_id, tag_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tagMeta.tagId}," + + " #{tagMeta.tagName}," + + " #{tagMeta.metalakeId}," + + " #{tagMeta.comment}," + + " #{tagMeta.properties}," + + " #{tagMeta.auditInfo}," + + " #{tagMeta.currentVersion}," + + " #{tagMeta.lastVersion}," + + " #{tagMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " tag_name = #{tagMeta.tagName}," + + " metalake_id = #{tagMeta.metalakeId}," + + " tag_comment = #{tagMeta.comment}," + + " properties = #{tagMeta.properties}," + + " audit_info = #{tagMeta.auditInfo}," + + " current_version = #{tagMeta.currentVersion}," + + " last_version = #{tagMeta.lastVersion}," + + " deleted_at = #{tagMeta.deletedAt}"; + } + + public String updateTagMeta( + @Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET tag_name = #{newTagMeta.tagName}," + + " tag_comment = #{newTagMeta.comment}," + + " properties = #{newTagMeta.properties}," + + " audit_info = #{newTagMeta.auditInfo}," + + " current_version = #{newTagMeta.currentVersion}," + + " last_version = #{newTagMeta.lastVersion}," + + " deleted_at = #{newTagMeta.deletedAt}" + + " WHERE tag_id = #{oldTagMeta.tagId}" + + " AND metalake_id = #{oldTagMeta.metalakeId}" + + " AND tag_name = #{oldTagMeta.tagName}" + + " AND (tag_comment IS NULL OR tag_comment = #{oldTagMeta.comment})" + + " AND properties = #{oldTagMeta.properties}" + + " AND audit_info = #{oldTagMeta.auditInfo}" + + " AND current_version = #{oldTagMeta.currentVersion}" + + " AND last_version = #{oldTagMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteTagMetaByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "UPDATE " + + TAG_TABLE_NAME + + " tm SET tm.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE tm.metalake_id IN (" + + " SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0"; + } + + public String softDeleteTagMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteTagMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TAG_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java index ad8afcfa366..a76a459f42d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -20,181 +20,54 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TagPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; public interface TagMetaMapper { String TAG_TABLE_NAME = "tag_meta"; - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId," - + " tm.tag_comment as comment," - + " tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider(type = TagMetaSQLProviderFactory.class, method = "listTagPOsByMetalake") List listTagPOsByMetalake(@Param("metalakeName") String metalakeName); - @Select( - "") + @SelectProvider( + type = TagMetaSQLProviderFactory.class, + method = "listTagPOsByMetalakeAndTagNames") List listTagPOsByMetalakeAndTagNames( @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames); - @Select( - "SELECT tm.tag_id as tagId FROM " - + TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" - + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider(type = TagMetaSQLProviderFactory.class, method = "selectTagIdByMetalakeAndName") Long selectTagIdByMetalakeAndName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId," - + " tm.tag_comment as comment," - + " tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" - + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider(type = TagMetaSQLProviderFactory.class, method = "selectTagMetaByMetalakeAndName") TagPO selectTagMetaByMetalakeAndName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Insert( - "INSERT INTO " - + TAG_TABLE_NAME - + " (tag_id, tag_name," - + " metalake_id, tag_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tagMeta.tagId}," - + " #{tagMeta.tagName}," - + " #{tagMeta.metalakeId}," - + " #{tagMeta.comment}," - + " #{tagMeta.properties}," - + " #{tagMeta.auditInfo}," - + " #{tagMeta.currentVersion}," - + " #{tagMeta.lastVersion}," - + " #{tagMeta.deletedAt}" - + " )") + @InsertProvider(type = TagMetaSQLProviderFactory.class, method = "insertTagMeta") void insertTagMeta(@Param("tagMeta") TagPO tagPO); - @Insert( - "INSERT INTO " - + TAG_TABLE_NAME - + "(tag_id, tag_name," - + " metalake_id, tag_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tagMeta.tagId}," - + " #{tagMeta.tagName}," - + " #{tagMeta.metalakeId}," - + " #{tagMeta.comment}," - + " #{tagMeta.properties}," - + " #{tagMeta.auditInfo}," - + " #{tagMeta.currentVersion}," - + " #{tagMeta.lastVersion}," - + " #{tagMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " tag_name = #{tagMeta.tagName}," - + " metalake_id = #{tagMeta.metalakeId}," - + " tag_comment = #{tagMeta.comment}," - + " properties = #{tagMeta.properties}," - + " audit_info = #{tagMeta.auditInfo}," - + " current_version = #{tagMeta.currentVersion}," - + " last_version = #{tagMeta.lastVersion}," - + " deleted_at = #{tagMeta.deletedAt}") + @InsertProvider( + type = TagMetaSQLProviderFactory.class, + method = "insertTagMetaOnDuplicateKeyUpdate") void insertTagMetaOnDuplicateKeyUpdate(@Param("tagMeta") TagPO tagPO); - @Update( - "UPDATE " - + TAG_TABLE_NAME - + " SET tag_name = #{newTagMeta.tagName}," - + " tag_comment = #{newTagMeta.comment}," - + " properties = #{newTagMeta.properties}," - + " audit_info = #{newTagMeta.auditInfo}," - + " current_version = #{newTagMeta.currentVersion}," - + " last_version = #{newTagMeta.lastVersion}," - + " deleted_at = #{newTagMeta.deletedAt}" - + " WHERE tag_id = #{oldTagMeta.tagId}" - + " AND metalake_id = #{oldTagMeta.metalakeId}" - + " AND tag_name = #{oldTagMeta.tagName}" - + " AND (tag_comment IS NULL OR tag_comment = #{oldTagMeta.comment})" - + " AND properties = #{oldTagMeta.properties}" - + " AND audit_info = #{oldTagMeta.auditInfo}" - + " AND current_version = #{oldTagMeta.currentVersion}" - + " AND last_version = #{oldTagMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = TagMetaSQLProviderFactory.class, method = "updateTagMeta") Integer updateTagMeta(@Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO); - @Update( - "UPDATE " - + TAG_TABLE_NAME - + " tm SET tm.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE tm.metalake_id IN (" - + " SELECT mm.metalake_id FROM " - + MetalakeMetaMapper.TABLE_NAME - + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0") + @UpdateProvider( + type = TagMetaSQLProviderFactory.class, + method = "softDeleteTagMetaByMetalakeAndTagName") Integer softDeleteTagMetaByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Update( - "UPDATE " - + TAG_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider(type = TagMetaSQLProviderFactory.class, method = "softDeleteTagMetasByMetalakeId") void softDeleteTagMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TAG_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider(type = TagMetaSQLProviderFactory.class, method = "deleteTagMetasByLegacyTimeline") Integer deleteTagMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java new file mode 100644 index 00000000000..aaa92b038ef --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.TagPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TagMetaSQLProviderFactory { + + private static final Map METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TagMetaMySQLProvider(), + JDBCBackendType.H2, new TagMetaH2Provider()); + + public static TagMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TagMetaMySQLProvider extends TagMetaBaseSQLProvider {} + + static class TagMetaH2Provider extends TagMetaBaseSQLProvider {} + + public static String listTagPOsByMetalake(@Param("metalakeName") String metalakeName) { + return getProvider().listTagPOsByMetalake(metalakeName); + } + + public static String listTagPOsByMetalakeAndTagNames( + @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames) { + return getProvider().listTagPOsByMetalakeAndTagNames(metalakeName, tagNames); + } + + public static String selectTagIdByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().selectTagIdByMetalakeAndName(metalakeName, tagName); + } + + public static String selectTagMetaByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().selectTagMetaByMetalakeAndName(metalakeName, tagName); + } + + public static String insertTagMeta(@Param("tagMeta") TagPO tagPO) { + return getProvider().insertTagMeta(tagPO); + } + + public static String insertTagMetaOnDuplicateKeyUpdate(@Param("tagMeta") TagPO tagPO) { + return getProvider().insertTagMetaOnDuplicateKeyUpdate(tagPO); + } + + public static String updateTagMeta( + @Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO) { + return getProvider().updateTagMeta(newTagPO, oldTagPO); + } + + public static String softDeleteTagMetaByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().softDeleteTagMetaByMetalakeAndTagName(metalakeName, tagName); + } + + public static String softDeleteTagMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTagMetasByMetalakeId(metalakeId); + } + + public static String deleteTagMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTagMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelBaseSQLProvider.java new file mode 100644 index 00000000000..9bcea1cd95e --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelBaseSQLProvider.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; +import org.apache.ibatis.annotations.Param; + +public class TagMetadataObjectRelBaseSQLProvider { + + public String listTagPOsByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND te.deleted_at = 0" + + " AND tm.deleted_at = 0"; + } + + public String getTagPOsByMetadataObjectAndTagName( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagName") String tagName) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0"; + } + + public String listTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," + + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," + + " te.current_version as currentVersion, te.last_version as lastVersion," + + " te.deleted_at as deletedAt" + + " FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te JOIN " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON te.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String batchInsertTagMetadataObjectRels( + @Param("tagRels") List tagRelPOs) { + return ""; + } + + public String batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagIds") List tagIds) { + return ""; + } + + public String softDeleteTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + public String softDeleteTagMetadataObjectRelsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE EXISTS (SELECT * FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + public String deleteTagEntityRelsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index 1ade27d8ffc..40b62955881 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -21,139 +21,63 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.po.TagPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; public interface TagMetadataObjectRelMapper { String TAG_METADATA_OBJECT_RELATION_TABLE_NAME = "tag_relation_meta"; - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm JOIN " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te ON tm.tag_id = te.tag_id" - + " WHERE te.metadata_object_id = #{metadataObjectId}" - + " AND te.metadata_object_type = #{metadataObjectType} AND te.deleted_at = 0" - + " AND tm.deleted_at = 0") + @SelectProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "listTagPOsByMetadataObjectIdAndType") List listTagPOsByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm JOIN " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te ON tm.tag_id = te.tag_id" - + " WHERE te.metadata_object_id = #{metadataObjectId}" - + " AND te.metadata_object_type = #{metadataObjectType} AND tm.tag_name = #{tagName}" - + " AND te.deleted_at = 0 AND tm.deleted_at = 0") + @SelectProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "getTagPOsByMetadataObjectAndTagName") TagPO getTagPOsByMetadataObjectAndTagName( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType, @Param("tagName") String tagName); - @Select( - "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," - + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," - + " te.current_version as currentVersion, te.last_version as lastVersion," - + " te.deleted_at as deletedAt" - + " FROM " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te JOIN " - + TagMetaMapper.TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON te.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" - + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "listTagMetadataObjectRelsByMetalakeAndTagName") List listTagMetadataObjectRelsByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Insert({ - "" - }) + @InsertProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "batchInsertTagMetadataObjectRels") void batchInsertTagMetadataObjectRels(@Param("tagRels") List tagRelPOs); - @Update({ - "" - }) + @UpdateProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject") void batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType, @Param("tagIds") List tagIds); - @Update( - "UPDATE " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " - + MetalakeMetaMapper.TABLE_NAME - + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.deleted_at = 0) AND te.deleted_at = 0") + @UpdateProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "softDeleteTagMetadataObjectRelsByMetalakeAndTagName") Integer softDeleteTagMetadataObjectRelsByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Update( - "UPDATE " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE EXISTS (SELECT * FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" - + " AND tm.deleted_at = 0) AND te.deleted_at = 0") + @UpdateProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "softDeleteTagMetadataObjectRelsByMetalakeId") void softDeleteTagMetadataObjectRelsByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "deleteTagEntityRelsByLegacyTimeline") Integer deleteTagEntityRelsByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java new file mode 100644 index 00000000000..b074349e817 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TagMetadataObjectRelSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TagMetadataObjectRelMySQLProvider(), + JDBCBackendType.H2, new TagMetadataObjectRelH2Provider()); + + public static TagMetadataObjectRelBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TagMetadataObjectRelMySQLProvider extends TagMetadataObjectRelBaseSQLProvider {} + + static class TagMetadataObjectRelH2Provider extends TagMetadataObjectRelBaseSQLProvider {} + + public static String listTagPOsByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider().listTagPOsByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String getTagPOsByMetadataObjectAndTagName( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagName") String tagName) { + return getProvider() + .getTagPOsByMetadataObjectAndTagName(metadataObjectId, metadataObjectType, tagName); + } + + public static String listTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().listTagMetadataObjectRelsByMetalakeAndTagName(metalakeName, tagName); + } + + public static String batchInsertTagMetadataObjectRels( + @Param("tagRels") List tagRelPOs) { + return getProvider().batchInsertTagMetadataObjectRels(tagRelPOs); + } + + public static String batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagIds") List tagIds) { + return getProvider() + .batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + metadataObjectId, metadataObjectType, tagIds); + } + + public static String softDeleteTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().softDeleteTagMetadataObjectRelsByMetalakeAndTagName(metalakeName, tagName); + } + + public static String softDeleteTagMetadataObjectRelsByMetalakeId( + @Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTagMetadataObjectRelsByMetalakeId(metalakeId); + } + + public static String deleteTagEntityRelsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTagEntityRelsByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaBaseSQLProvider.java new file mode 100644 index 00000000000..a5db8e0f943 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaBaseSQLProvider.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.UserMetaMapper.USER_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.UserPO; +import org.apache.ibatis.annotations.Param; + +public class UserMetaBaseSQLProvider { + + public String selectUserIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name) { + return "SELECT user_id as userId FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" + + " AND deleted_at = 0"; + } + + public String selectUserMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name) { + return "SELECT user_id as userId, user_name as userName," + + " metalake_id as metalakeId," + + " audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" + + " AND deleted_at = 0"; + } + + public String insertUserMeta(@Param("userMeta") UserPO userPO) { + return "INSERT INTO " + + USER_TABLE_NAME + + "(user_id, user_name," + + " metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )"; + } + + public String insertUserMetaOnDuplicateKeyUpdate(@Param("userMeta") UserPO userPO) { + return "INSERT INTO " + + USER_TABLE_NAME + + "(user_id, user_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " user_name = #{userMeta.userName}," + + " metalake_id = #{userMeta.metalakeId}," + + " audit_info = #{userMeta.auditInfo}," + + " current_version = #{userMeta.currentVersion}," + + " last_version = #{userMeta.lastVersion}," + + " deleted_at = #{userMeta.deletedAt}"; + } + + public String softDeleteUserMetaByUserId(@Param("userId") Long userId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + public String softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String updateUserMeta( + @Param("newUserMeta") UserPO newUserPO, @Param("oldUserMeta") UserPO oldUserPO) { + return "UPDATE " + + USER_TABLE_NAME + + " SET user_name = #{newUserMeta.userName}," + + " metalake_id = #{newUserMeta.metalakeId}," + + " audit_info = #{newUserMeta.auditInfo}," + + " current_version = #{newUserMeta.currentVersion}," + + " last_version = #{newUserMeta.lastVersion}," + + " deleted_at = #{newUserMeta.deletedAt}" + + " WHERE user_id = #{oldUserMeta.userId}" + + " AND user_name = #{oldUserMeta.userName}" + + " AND metalake_id = #{oldUserMeta.metalakeId}" + + " AND audit_info = #{oldUserMeta.auditInfo}" + + " AND current_version = #{oldUserMeta.currentVersion}" + + " AND last_version = #{oldUserMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String listUsersByRoleId(@Param("roleId") Long roleId) { + return "SELECT us.user_id as userId, us.user_name as userName," + + " us.metalake_id as metalakeId," + + " us.audit_info as auditInfo, us.current_version as currentVersion," + + " us.last_version as lastVersion, us.deleted_at as deletedAt" + + " FROM " + + USER_TABLE_NAME + + " us JOIN " + + USER_ROLE_RELATION_TABLE_NAME + + " re ON us.user_id = re.user_id" + + " WHERE re.role_id = #{roleId}" + + " AND us.deleted_at = 0 AND re.deleted_at = 0"; + } + + public String deleteUserMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + USER_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java index e7b442c099a..ad794c39530 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.UserPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -39,121 +39,44 @@ public interface UserMetaMapper { String USER_TABLE_NAME = "user_meta"; String USER_ROLE_RELATION_TABLE_NAME = "user_role_rel"; - @Select( - "SELECT user_id as userId FROM " - + USER_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" - + " AND deleted_at = 0") + @SelectProvider( + type = UserMetaSQLProviderFactory.class, + method = "selectUserIdByMetalakeIdAndName") Long selectUserIdByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("userName") String name); - @Select( - "SELECT user_id as userId, user_name as userName," - + " metalake_id as metalakeId," - + " audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + USER_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" - + " AND deleted_at = 0") + @SelectProvider( + type = UserMetaSQLProviderFactory.class, + method = "selectUserMetaByMetalakeIdAndName") UserPO selectUserMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("userName") String name); - @Insert( - "INSERT INTO " - + USER_TABLE_NAME - + "(user_id, user_name," - + " metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{userMeta.userId}," - + " #{userMeta.userName}," - + " #{userMeta.metalakeId}," - + " #{userMeta.auditInfo}," - + " #{userMeta.currentVersion}," - + " #{userMeta.lastVersion}," - + " #{userMeta.deletedAt}" - + " )") + @InsertProvider(type = UserMetaSQLProviderFactory.class, method = "insertUserMeta") void insertUserMeta(@Param("userMeta") UserPO userPO); - @Insert( - "INSERT INTO " - + USER_TABLE_NAME - + "(user_id, user_name," - + "metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{userMeta.userId}," - + " #{userMeta.userName}," - + " #{userMeta.metalakeId}," - + " #{userMeta.auditInfo}," - + " #{userMeta.currentVersion}," - + " #{userMeta.lastVersion}," - + " #{userMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " user_name = #{userMeta.userName}," - + " metalake_id = #{userMeta.metalakeId}," - + " audit_info = #{userMeta.auditInfo}," - + " current_version = #{userMeta.currentVersion}," - + " last_version = #{userMeta.lastVersion}," - + " deleted_at = #{userMeta.deletedAt}") + @InsertProvider( + type = UserMetaSQLProviderFactory.class, + method = "insertUserMetaOnDuplicateKeyUpdate") void insertUserMetaOnDuplicateKeyUpdate(@Param("userMeta") UserPO userPO); - @Update( - "UPDATE " - + USER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE user_id = #{userId} AND deleted_at = 0") + @UpdateProvider(type = UserMetaSQLProviderFactory.class, method = "softDeleteUserMetaByUserId") void softDeleteUserMetaByUserId(@Param("userId") Long userId); - @Update( - "UPDATE " - + USER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = UserMetaSQLProviderFactory.class, + method = "softDeleteUserMetasByMetalakeId") void softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + USER_TABLE_NAME - + " SET user_name = #{newUserMeta.userName}," - + " metalake_id = #{newUserMeta.metalakeId}," - + " audit_info = #{newUserMeta.auditInfo}," - + " current_version = #{newUserMeta.currentVersion}," - + " last_version = #{newUserMeta.lastVersion}," - + " deleted_at = #{newUserMeta.deletedAt}" - + " WHERE user_id = #{oldUserMeta.userId}" - + " AND user_name = #{oldUserMeta.userName}" - + " AND metalake_id = #{oldUserMeta.metalakeId}" - + " AND audit_info = #{oldUserMeta.auditInfo}" - + " AND current_version = #{oldUserMeta.currentVersion}" - + " AND last_version = #{oldUserMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = UserMetaSQLProviderFactory.class, method = "updateUserMeta") Integer updateUserMeta( @Param("newUserMeta") UserPO newUserPO, @Param("oldUserMeta") UserPO oldUserPO); - @Select( - "SELECT us.user_id as userId, us.user_name as userName," - + " us.metalake_id as metalakeId," - + " us.audit_info as auditInfo, us.current_version as currentVersion," - + " us.last_version as lastVersion, us.deleted_at as deletedAt" - + " FROM " - + USER_TABLE_NAME - + " us JOIN " - + USER_ROLE_RELATION_TABLE_NAME - + " re ON us.user_id = re.user_id" - + " WHERE re.role_id = #{roleId}" - + " AND us.deleted_at = 0 AND re.deleted_at = 0") + @SelectProvider(type = UserMetaSQLProviderFactory.class, method = "listUsersByRoleId") List listUsersByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + USER_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = UserMetaSQLProviderFactory.class, + method = "deleteUserMetasByLegacyTimeline") Integer deleteUserMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java new file mode 100644 index 00000000000..3c64f510c27 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.UserPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class UserMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new UserMetaMySQLProvider(), + JDBCBackendType.H2, new UserMetaH2Provider()); + + public static UserMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class UserMetaMySQLProvider extends UserMetaBaseSQLProvider {} + + static class UserMetaH2Provider extends UserMetaBaseSQLProvider {} + + public static String selectUserIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String userName) { + return getProvider().selectUserIdByMetalakeIdAndName(metalakeId, userName); + } + + public static String selectUserMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name) { + return getProvider().selectUserMetaByMetalakeIdAndName(metalakeId, name); + } + + public static String insertUserMeta(@Param("userMeta") UserPO userPO) { + return getProvider().insertUserMeta(userPO); + } + + public static String insertUserMetaOnDuplicateKeyUpdate(@Param("userMeta") UserPO userPO) { + return getProvider().insertUserMetaOnDuplicateKeyUpdate(userPO); + } + + public static String softDeleteUserMetaByUserId(@Param("userId") Long userId) { + return getProvider().softDeleteUserMetaByUserId(userId); + } + + public static String softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteUserMetasByMetalakeId(metalakeId); + } + + public static String updateUserMeta( + @Param("newUserMeta") UserPO newUserPO, @Param("oldUserMeta") UserPO oldUserPO) { + return getProvider().updateUserMeta(newUserPO, oldUserPO); + } + + public static String listUsersByRoleId(@Param("roleId") Long roleId) { + return getProvider().listUsersByRoleId(roleId); + } + + public static String deleteUserMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteUserMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelBaseSQLProvider.java new file mode 100644 index 00000000000..a1e6659182d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelBaseSQLProvider.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.UserRoleRelPO; +import org.apache.ibatis.annotations.Param; + +public class UserRoleRelBaseSQLProvider { + + public String batchInsertUserRoleRel( + @Param("userRoleRelList") List userRoleRelList) { + return ""; + } + + public String batchInsertUserRoleRelOnDuplicateKeyUpdate( + @Param("userRoleRels") List userRoleRelPOs) { + return ""; + } + + public String softDeleteUserRoleRelByUserId(@Param("userId") Long userId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " + + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + public String softDeleteUserRoleRelByUserAndRoles( + @Param("userId") Long userId, @Param("roleIds") List roleIds) { + return ""; + } + + public String softDeleteUserRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " + + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE user_id IN (SELECT user_id FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + public String softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " + + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String deleteUserRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + USER_ROLE_RELATION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java index 23c9d6702cb..5e540de8fe2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java @@ -21,10 +21,10 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.UserRoleRelPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -38,101 +38,39 @@ public interface UserRoleRelMapper { String USER_TABLE_NAME = "user_meta"; String USER_ROLE_RELATION_TABLE_NAME = "user_role_rel"; - @Insert({ - "" - }) + @InsertProvider(type = UserRoleRelSQLProviderFactory.class, method = "batchInsertUserRoleRel") void batchInsertUserRoleRel(@Param("userRoleRels") List userRoleRelPOs); - @Insert({ - "" - }) + @InsertProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "batchInsertUserRoleRelOnDuplicateKeyUpdate") void batchInsertUserRoleRelOnDuplicateKeyUpdate( @Param("userRoleRels") List userRoleRelPOs); - @Update( - "UPDATE " - + USER_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " - + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE user_id = #{userId} AND deleted_at = 0") + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByUserId") void softDeleteUserRoleRelByUserId(@Param("userId") Long userId); - @Update({ - "" - }) + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByUserAndRoles") void softDeleteUserRoleRelByUserAndRoles( @Param("userId") Long userId, @Param("roleIds") List roleIds); - @Update( - "UPDATE " - + USER_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " - + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE user_id IN (SELECT user_id FROM " - + USER_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" - + " AND deleted_at = 0") - void softDeleteUserRoleRelByMetalakeId(Long metalakeId); + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByMetalakeId") + void softDeleteUserRoleRelByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + USER_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " - + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByRoleId") void softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + USER_ROLE_RELATION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "deleteUserRoleRelMetasByLegacyTimeline") Integer deleteUserRoleRelMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java new file mode 100644 index 00000000000..3d52c6aa02c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.po.UserRoleRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class UserRoleRelSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new UserRoleRelMySQLProvider(), + JDBCBackendType.H2, new UserRoleRelH2Provider()); + + public static UserRoleRelBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class UserRoleRelMySQLProvider extends UserRoleRelBaseSQLProvider {} + + static class UserRoleRelH2Provider extends UserRoleRelBaseSQLProvider {} + + public static String batchInsertUserRoleRel( + @Param("userRoleRels") List userRoleRelPOs) { + return getProvider().batchInsertUserRoleRel(userRoleRelPOs); + } + + public static String batchInsertUserRoleRelOnDuplicateKeyUpdate( + @Param("userRoleRels") List userRoleRelPOs) { + return getProvider().batchInsertUserRoleRelOnDuplicateKeyUpdate(userRoleRelPOs); + } + + public static String softDeleteUserRoleRelByUserId(@Param("userId") Long userId) { + return getProvider().softDeleteUserRoleRelByUserId(userId); + } + + public static String softDeleteUserRoleRelByUserAndRoles( + @Param("userId") Long userId, @Param("roleIds") List roleIds) { + return getProvider().softDeleteUserRoleRelByUserAndRoles(userId, roleIds); + } + + public static String softDeleteUserRoleRelByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteUserRoleRelByMetalakeId(metalakeId); + } + + public static String softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId) { + return getProvider().softDeleteUserRoleRelByRoleId(roleId); + } + + public static String deleteUserRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteUserRoleRelMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index f89c4c5f0f0..3e116781162 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -211,7 +211,8 @@ public boolean deleteSchema(NameIdentifier identifier, boolean cascade) { mapper -> mapper.softDeleteTopicMetasBySchemaId(schemaId)), () -> SessionUtils.doWithoutCommit( - OwnerMetaMapper.class, mapper -> mapper.sotDeleteOwnerRelBySchemaId(schemaId))); + OwnerMetaMapper.class, + mapper -> mapper.softDeleteOwnerRelBySchemaId(schemaId))); } else { List tableEntities = TableMetaService.getInstance() From 3731e53a8a249812602ab9181b17a497d6dde567 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Sat, 7 Sep 2024 00:06:46 +0800 Subject: [PATCH 4/8] [#4708] improvement(client-java/server): Add implementations for the `getFileLocation` interface in Java Client / Server (#4858) ### What changes were proposed in this pull request? Add the implementations for `getFileLocation` interfaces in Java Client / Server. ### Why are the changes needed? Fix: #4708 ### How was this patch tested? Add some UTs and ITs. --------- Co-authored-by: xiaojiebao --- .../gravitino/client/FilesetCatalog.java | 34 ++++- .../gravitino/client/TestFilesetCatalog.java | 136 ++++++++++++++++++ docs/open-api/filesets.yaml | 65 +++++++++ .../apache/gravitino/server/web/Utils.java | 27 ++++ .../server/web/rest/FilesetOperations.java | 45 ++++++ .../gravitino/server/web/TestUtils.java | 38 +++++ .../web/rest/TestFilesetOperations.java | 49 +++++++ 7 files changed, 393 insertions(+), 1 deletion(-) diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java index f568370b139..e3b2aa9a6cf 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -29,6 +30,7 @@ import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.audit.CallerContext; import org.apache.gravitino.dto.AuditDTO; import org.apache.gravitino.dto.CatalogDTO; import org.apache.gravitino.dto.requests.FilesetCreateRequest; @@ -36,6 +38,7 @@ import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; @@ -238,7 +241,23 @@ public boolean dropFileset(NameIdentifier ident) { @Override public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { - throw new UnsupportedOperationException("Not implemented"); + checkFilesetNameIdentifier(ident); + Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); + + CallerContext callerContext = CallerContext.CallerContextHolder.get(); + + Map params = new HashMap<>(); + params.put("sub_path", RESTUtils.encodeString(subPath)); + FileLocationResponse resp = + restClient.get( + formatFileLocationRequestPath(fullNamespace, ident.name()), + params, + FileLocationResponse.class, + callerContext != null ? callerContext.context() : Collections.emptyMap(), + ErrorHandlers.filesetErrorHandler()); + resp.validate(); + + return resp.getFileLocation(); } @VisibleForTesting @@ -252,6 +271,19 @@ static String formatFilesetRequestPath(Namespace ns) { .toString(); } + @VisibleForTesting + static String formatFileLocationRequestPath(Namespace ns, String name) { + Namespace schemaNs = Namespace.of(ns.level(0), ns.level(1)); + return new StringBuilder() + .append(formatSchemaRequestPath(schemaNs)) + .append("/") + .append(RESTUtils.encodeString(ns.level(2))) + .append("/filesets/") + .append(RESTUtils.encodeString(name)) + .append("/location") + .toString(); + } + /** * Check whether the namespace of a fileset is valid. * diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java index eee24ee3f21..8a87fb72e96 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java @@ -28,10 +28,19 @@ import com.google.common.collect.ImmutableMap; import java.nio.file.NoSuchFileException; import java.time.Instant; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.audit.CallerContext; +import org.apache.gravitino.audit.FilesetAuditConstants; +import org.apache.gravitino.audit.FilesetDataOperation; +import org.apache.gravitino.audit.InternalClientType; import org.apache.gravitino.dto.AuditDTO; import org.apache.gravitino.dto.CatalogDTO; import org.apache.gravitino.dto.file.FilesetDTO; @@ -43,6 +52,7 @@ import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; @@ -50,10 +60,15 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.rest.RESTUtils; import org.apache.hc.core5.http.Method; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.mockserver.matchers.Times; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; +import org.mockserver.model.Parameter; public class TestFilesetCatalog extends TestBase { @@ -400,6 +415,127 @@ public void testAlterFileset() throws JsonProcessingException { "internal error"); } + @Test + public void testGetFileLocation() throws JsonProcessingException { + NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + String mockSubPath = "mock_location/test"; + String filesetPath = + withSlash( + FilesetCatalog.formatFileLocationRequestPath( + Namespace.of(metalakeName, catalogName, "schema1"), fileset.name())); + Map queryParams = new HashMap<>(); + queryParams.put("sub_path", RESTUtils.encodeString(mockSubPath)); + + String mockFileLocation = + String.format("file:/fileset/%s/%s/%s/%s", catalogName, "schema1", "fileset1", mockSubPath); + FileLocationResponse resp = new FileLocationResponse(mockFileLocation); + buildMockResource(Method.GET, filesetPath, queryParams, null, resp, SC_OK); + + String actualFileLocation = + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath); + Assertions.assertTrue(StringUtils.isNotBlank(actualFileLocation)); + Assertions.assertEquals(mockFileLocation, actualFileLocation); + + // Throw schema not found exception + ErrorResponse errResp = + ErrorResponse.notFound(NoSuchSchemaException.class.getSimpleName(), "schema not found"); + buildMockResource(Method.GET, filesetPath, null, errResp, SC_NOT_FOUND); + Assertions.assertThrows( + NoSuchSchemaException.class, + () -> + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath), + "schema not found"); + + ErrorResponse errResp1 = + ErrorResponse.notFound(NotFoundException.class.getSimpleName(), "fileset not found"); + buildMockResource(Method.GET, filesetPath, null, errResp1, SC_NOT_FOUND); + Assertions.assertThrows( + NotFoundException.class, + () -> + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath), + "fileset not found"); + + ErrorResponse errResp2 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.GET, filesetPath, null, errResp2, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, + () -> + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath), + "internal error"); + } + + @Test + public void testCallerContextToHeader() throws JsonProcessingException { + NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + String mockSubPath = "mock_location/test"; + String filesetPath = + withSlash( + FilesetCatalog.formatFileLocationRequestPath( + Namespace.of(metalakeName, catalogName, "schema1"), fileset.name())); + Map queryParams = new HashMap<>(); + queryParams.put("sub_path", RESTUtils.encodeString(mockSubPath)); + String mockFileLocation = + String.format("file:/fileset/%s/%s/%s/%s", catalogName, "schema1", "fileset1", mockSubPath); + FileLocationResponse resp = new FileLocationResponse(mockFileLocation); + String respJson = MAPPER.writeValueAsString(resp); + + List parameters = + queryParams.entrySet().stream() + .map(kv -> new Parameter(kv.getKey(), kv.getValue())) + .collect(Collectors.toList()); + HttpRequest mockRequest = + HttpRequest.request(filesetPath) + .withMethod(Method.GET.name()) + .withQueryStringParameters(parameters); + HttpResponse mockResponse = HttpResponse.response().withStatusCode(SC_OK).withBody(respJson); + + // set the thread local context + Map context = new HashMap<>(); + context.put( + FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, + InternalClientType.HADOOP_GVFS.name()); + context.put( + FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION, + FilesetDataOperation.GET_FILE_STATUS.name()); + CallerContext callerContext = CallerContext.builder().withContext(context).build(); + CallerContext.CallerContextHolder.set(callerContext); + + // Using Times.exactly(1) will only match once for the request, so we could set difference + // responses for the same request and path. + AtomicReference internalClientType = new AtomicReference<>(null); + AtomicReference dataOperation = new AtomicReference<>(null); + mockServer + .when(mockRequest, Times.exactly(1)) + .respond( + httpRequest -> { + internalClientType.set( + httpRequest.getFirstHeader( + FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); + dataOperation.set( + httpRequest.getFirstHeader( + FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); + return mockResponse; + }); + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath); + Assertions.assertEquals(FilesetDataOperation.GET_FILE_STATUS.name(), dataOperation.get()); + Assertions.assertEquals(InternalClientType.HADOOP_GVFS.name(), internalClientType.get()); + } + private FilesetDTO mockFilesetDTO( String name, Fileset.Type type, diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml index 865324838a0..f2b3fe83598 100644 --- a/docs/open-api/filesets.yaml +++ b/docs/open-api/filesets.yaml @@ -157,6 +157,46 @@ paths: "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/filesets/{fileset}/location: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/fileset" + + get: + tags: + - location + summary: Get file location + operationId: getFileLocation + description: Returns the specified file location object + parameters: + - name: sub_path + in: query + required: true + schema: + type: string + description: The sub path to the file or directory + responses: + "200": + $ref: "#/components/responses/FileLocationResponse" + "404": + description: Not Found - The target fileset does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchCatalogException: + $ref: "./catalogs.yaml#/components/examples/NoSuchCatalogException" + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + NoSuchFilesetException: + $ref: "#/components/examples/NoSuchFilesetException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" components: @@ -356,6 +396,25 @@ components: FilesetResponse: $ref: "#/components/examples/FilesetResponse" + FileLocationResponse: + description: The response of the file location object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + fileLocation: + type: string + description: The actual file location + examples: + FileLocationResponse: + $ref: "#/components/examples/FileLocationResponse" examples: FilesetCreateRequest: @@ -385,6 +444,12 @@ components: } } + FileLocationResponse: + value: { + "code": 0, + "fileLocation": "hdfs://host/user/fileset/schema/fileset1/test.parquet" + } + FilesetAlreadyExistsException: value: { "code": 1004, diff --git a/server/src/main/java/org/apache/gravitino/server/web/Utils.java b/server/src/main/java/org/apache/gravitino/server/web/Utils.java index e2405a6adff..3503f90dd9a 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/Utils.java +++ b/server/src/main/java/org/apache/gravitino/server/web/Utils.java @@ -18,12 +18,18 @@ */ package org.apache.gravitino.server.web; +import com.google.common.collect.Maps; import java.security.PrivilegedExceptionAction; +import java.util.Map; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.UserPrincipal; +import org.apache.gravitino.audit.FilesetAuditConstants; +import org.apache.gravitino.audit.FilesetDataOperation; +import org.apache.gravitino.audit.InternalClientType; import org.apache.gravitino.auth.AuthConstants; import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.utils.PrincipalUtils; @@ -148,4 +154,25 @@ public static Response doAs( } return PrincipalUtils.doAs(principal, action); } + + public static Map filterFilesetAuditHeaders(HttpServletRequest httpRequest) { + Map filteredHeaders = Maps.newHashMap(); + + String internalClientType = + httpRequest.getHeader(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE); + if (StringUtils.isNotBlank(internalClientType) + && InternalClientType.checkValid(internalClientType)) { + filteredHeaders.put( + FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, internalClientType); + } + + String dataOperation = + httpRequest.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION); + if (StringUtils.isNotBlank( + httpRequest.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)) + && FilesetDataOperation.checkValid(dataOperation)) { + filteredHeaders.put(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION, dataOperation); + } + return filteredHeaders; + } } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java index 0d6f1fde34b..f7838d0ae83 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java @@ -20,9 +20,11 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; +import java.util.Map; import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; +import javax.validation.constraints.NotNull; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -30,16 +32,19 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.audit.CallerContext; import org.apache.gravitino.catalog.FilesetDispatcher; import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.dto.util.DTOConverters; import org.apache.gravitino.file.Fileset; @@ -245,4 +250,44 @@ public Response dropFileset( return ExceptionHandlers.handleFilesetException(OperationType.DROP, fileset, schema, e); } } + + @GET + @Path("{fileset}/location") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-file-location." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-file-location", absolute = true) + public Response getFileLocation( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("fileset") String fileset, + @QueryParam("sub_path") @NotNull String subPath) { + LOG.info( + "Received get file location request: {}.{}.{}.{}, sub path:{}", + metalake, + catalog, + schema, + fileset, + subPath); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifierUtil.ofFileset(metalake, catalog, schema, fileset); + Map filteredAuditHeaders = Utils.filterFilesetAuditHeaders(httpRequest); + // set the audit info into the thread local context + if (!filteredAuditHeaders.isEmpty()) { + CallerContext context = + CallerContext.builder().withContext(filteredAuditHeaders).build(); + CallerContext.CallerContextHolder.set(context); + } + String actualFileLocation = + TreeLockUtils.doWithTreeLock( + ident, LockType.READ, () -> dispatcher.getFileLocation(ident, subPath)); + return Utils.ok(new FileLocationResponse(actualFileLocation)); + }); + } catch (Exception e) { + return ExceptionHandlers.handleFilesetException(OperationType.GET, fileset, schema, e); + } + } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java b/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java index adaaacaed09..b77fa75e4f7 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java +++ b/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java @@ -23,11 +23,17 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.gravitino.audit.FilesetAuditConstants; +import org.apache.gravitino.audit.FilesetDataOperation; +import org.apache.gravitino.audit.InternalClientType; import org.apache.gravitino.dto.responses.ErrorResponse; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -167,4 +173,36 @@ public void testUnsupportedOperation() { ErrorResponse errorResponse = (ErrorResponse) response.getEntity(); assertEquals("Unsupported operation", errorResponse.getMessage()); } + + @Test + public void testFilterFilesetAuditHeaders() { + // test invalid internal client type + HttpServletRequest mockRequest1 = Mockito.mock(HttpServletRequest.class); + when(mockRequest1.getHeader(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)) + .thenReturn("test"); + Assertions.assertThrows( + IllegalArgumentException.class, () -> Utils.filterFilesetAuditHeaders(mockRequest1)); + + // test invalid fileset data operation + HttpServletRequest mockRequest2 = Mockito.mock(HttpServletRequest.class); + when(mockRequest2.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)) + .thenReturn("test"); + Assertions.assertThrows( + IllegalArgumentException.class, () -> Utils.filterFilesetAuditHeaders(mockRequest2)); + + // test normal audit headers + HttpServletRequest mockRequest3 = Mockito.mock(HttpServletRequest.class); + when(mockRequest3.getHeader(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)) + .thenReturn(InternalClientType.HADOOP_GVFS.name()); + when(mockRequest3.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)) + .thenReturn(FilesetDataOperation.GET_FILE_STATUS.name()); + Map filteredMap = Utils.filterFilesetAuditHeaders(mockRequest3); + Assertions.assertEquals(2, filteredMap.size()); + Assertions.assertEquals( + InternalClientType.HADOOP_GVFS.name(), + filteredMap.get(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); + Assertions.assertEquals( + FilesetDataOperation.GET_FILE_STATUS.name(), + filteredMap.get(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); + } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java index 1492b8aa69a..c1badd3fb1f 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java @@ -51,6 +51,7 @@ import org.apache.gravitino.dto.responses.EntityListResponse; import org.apache.gravitino.dto.responses.ErrorConstants; import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; @@ -432,6 +433,54 @@ public void testDropFileset() { Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp.getType()); } + @Test + public void testGetFileLocation() { + String actualPath = "mock location/path1"; + + when(dispatcher.getFileLocation(any(), any())).thenReturn(actualPath); + Response resp = + target(filesetPath(metalake, catalog, schema) + "fileset1/location") + .queryParam("sub_path", RESTUtils.encodeString("test/1")) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + + FileLocationResponse contextResponse = resp.readEntity(FileLocationResponse.class); + Assertions.assertEquals(0, contextResponse.getCode()); + + Assertions.assertEquals(actualPath, contextResponse.getFileLocation()); + + // Test throw NoSuchFilesetException + doThrow(new NoSuchFilesetException("no found")).when(dispatcher).getFileLocation(any(), any()); + Response resp1 = + target(filesetPath(metalake, catalog, schema) + "fileset1/location") + .queryParam("sub_path", RESTUtils.encodeString("test/1")) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchFilesetException.class.getSimpleName(), errorResp.getType()); + + // Test throw RuntimeException + doThrow(new RuntimeException("internal error")).when(dispatcher).getFileLocation(any(), any()); + Response resp2 = + target(filesetPath(metalake, catalog, schema) + "fileset1/location") + .queryParam("sub_path", RESTUtils.encodeString("test/1")) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp2 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + private void assertUpdateFileset(FilesetUpdatesRequest req, Fileset updatedFileset) { when(dispatcher.alterFileset(any(), any(FilesetChange.class))).thenReturn(updatedFileset); From 3ba729f249806c89d6a0e02d51e1fe688db15ac4 Mon Sep 17 00:00:00 2001 From: roryqi Date: Sat, 7 Sep 2024 00:24:02 +0800 Subject: [PATCH 5/8] [#4608] feat(core): Supports to load fields lazily (#4690) ### What changes were proposed in this pull request? Supports to load fields lazily. ### Why are the changes needed? Fix: #4608 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add new uts and pass the existing uts --- .../ranger/RangerAuthorizationPlugin.java | 6 +- .../ranger/integration/test/RangerHiveIT.java | 42 ++---- .../authorization/PermissionManager.java | 82 +++++------ .../authorization/UserGroupManager.java | 4 +- .../apache/gravitino/meta/GroupEntity.java | 58 ++++---- .../org/apache/gravitino/meta/RoleEntity.java | 41 +++--- .../org/apache/gravitino/meta/UserEntity.java | 58 ++++---- .../gravitino/proto/GroupEntitySerDe.java | 69 ---------- .../gravitino/proto/ProtoEntitySerDe.java | 13 +- .../gravitino/proto/RoleEntitySerDe.java | 115 ---------------- .../gravitino/proto/UserEntitySerDe.java | 69 ---------- .../storage/kv/BinaryEntityKeyEncoder.java | 9 -- .../relational/service/GroupMetaService.java | 26 ++-- .../relational/service/OwnerMetaService.java | 6 +- .../relational/service/RoleMetaService.java | 42 +----- .../relational/service/UserMetaService.java | 25 ++-- .../relational/utils/POConverters.java | 63 +++++---- .../relational/utils/SupplierUtils.java | 114 ++++++++++++++++ ...estAccessControlManagerForPermissions.java | 2 +- .../authorization/TestOwnerManager.java | 6 +- .../org/apache/gravitino/meta/TestEntity.java | 98 +++++++++++++- .../gravitino/proto/TestEntityProtoSerDe.java | 113 ---------------- .../gravitino/storage/TestEntityStorage.java | 5 +- .../storage/memory/TestMemoryEntityStore.java | 5 +- .../storage/relational/TestJDBCBackend.java | 35 ++--- .../service/TestGroupMetaService.java | 126 +++++++---------- .../service/TestRoleMetaService.java | 35 ++--- .../service/TestUserMetaService.java | 127 +++++++----------- .../server/web/rest/TestGroupOperations.java | 2 +- .../web/rest/TestPermissionOperations.java | 27 ++-- .../server/web/rest/TestUserOperations.java | 2 +- 31 files changed, 555 insertions(+), 870 deletions(-) delete mode 100644 core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java delete mode 100644 core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/utils/SupplierUtils.java diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 75692e987b8..03e18bef914 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -194,8 +194,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n UserEntity.builder() .withId(1L) .withName(newOwner.name()) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); onUserAdded(userEntity); @@ -204,8 +203,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n GroupEntity.builder() .withId(1L) .withName(newOwner.name()) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); onGroupAdded(groupEntity); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index 79f66ef287a..844b89b3008 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -483,8 +483,7 @@ public void testOnGrantedRolesToUser() { UserEntity.builder() .withId(1L) .withName(userName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -502,8 +501,7 @@ public void testOnGrantedRolesToUser() { UserEntity.builder() .withId(1L) .withName(userName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -525,8 +523,7 @@ public void testOnRevokedRolesFromUser() { UserEntity.builder() .withId(1L) .withName(userName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -555,8 +552,7 @@ public void testOnGrantedRolesToGroup() { GroupEntity.builder() .withId(1L) .withName(groupName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -574,8 +570,7 @@ public void testOnGrantedRolesToGroup() { GroupEntity.builder() .withId(1L) .withName(groupName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -598,8 +593,7 @@ public void testOnRevokedRolesFromGroup() { GroupEntity.builder() .withId(1L) .withName(groupName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -680,8 +674,7 @@ public void testCreateUser() { .withId(0L) .withName(currentFunName()) .withAuditInfo(auditInfo) - .withRoleIds(null) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); Assertions.assertTrue(rangerAuthPlugin.onUserAdded(user)); Assertions.assertTrue(rangerAuthPlugin.onUserAcquired(user)); @@ -696,8 +689,7 @@ public void testCreateGroup() { .withId(0L) .withName(currentFunName()) .withAuditInfo(auditInfo) - .withRoleIds(null) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); Assertions.assertTrue(rangerAuthPlugin.onGroupAdded(group)); @@ -769,8 +761,7 @@ public void testCombinationOperation() { UserEntity.builder() .withId(1L) .withName(userName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -786,8 +777,7 @@ public void testCombinationOperation() { UserEntity.builder() .withId(1L) .withName(userName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -800,8 +790,7 @@ public void testCombinationOperation() { UserEntity.builder() .withId(1L) .withName(userName3) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -833,8 +822,7 @@ public void testCombinationOperation() { GroupEntity.builder() .withId(1L) .withName(groupName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -852,8 +840,7 @@ public void testCombinationOperation() { GroupEntity.builder() .withId(1L) .withName(groupName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -871,8 +858,7 @@ public void testCombinationOperation() { GroupEntity.builder() .withId(1L) .withName(groupName3) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( diff --git a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java index edb02cdcec5..479bb069e3f 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java @@ -22,9 +22,11 @@ import static org.apache.gravitino.authorization.AuthorizationUtils.USER_DOES_NOT_EXIST_MSG; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -70,14 +72,10 @@ User grantRolesToUser(String metalake, List roles, String user) { UserEntity.class, Entity.EntityType.USER, userEntity -> { - List roleEntities = Lists.newArrayList(); - if (userEntity.roleNames() != null) { - for (String role : userEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } - } - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); - List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + List roleEntities = Lists.newArrayList(userEntity.roleEntities()); + + List roleIds = + roleEntities.stream().map(RoleEntity::id).collect(Collectors.toList()); for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { if (roleIds.contains(roleEntityToGrant.id())) { @@ -87,8 +85,8 @@ User grantRolesToUser(String metalake, List roles, String user) { user, metalake); } else { - roleNames.add(roleEntityToGrant.name()); roleIds.add(roleEntityToGrant.id()); + roleEntities.add(roleEntityToGrant); } } @@ -104,8 +102,7 @@ User grantRolesToUser(String metalake, List roles, String user) { .withNamespace(userEntity.namespace()) .withId(userEntity.id()) .withName(userEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(auditInfo) .build(); }); @@ -149,13 +146,8 @@ Group grantRolesToGroup(String metalake, List roles, String group) { GroupEntity.class, Entity.EntityType.GROUP, groupEntity -> { - List roleEntities = Lists.newArrayList(); - if (groupEntity.roleNames() != null) { - for (String role : groupEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } - } - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); + List roleEntities = Lists.newArrayList(groupEntity.roleEntities()); + List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { @@ -166,8 +158,8 @@ Group grantRolesToGroup(String metalake, List roles, String group) { group, metalake); } else { - roleNames.add(roleEntityToGrant.name()); roleIds.add(roleEntityToGrant.id()); + roleEntities.add(roleEntityToGrant); } } @@ -183,8 +175,7 @@ Group grantRolesToGroup(String metalake, List roles, String group) { .withId(groupEntity.id()) .withNamespace(groupEntity.namespace()) .withName(groupEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(auditInfo) .build(); }); @@ -228,19 +219,18 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) { GroupEntity.class, Entity.EntityType.GROUP, groupEntity -> { - List roleEntities = Lists.newArrayList(); - if (groupEntity.roleNames() != null) { - for (String role : groupEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } + List roleEntities = groupEntity.roleEntities(); + + // Avoid loading securable objects + Map roleEntitiesMap = Maps.newHashMap(); + for (RoleEntity entity : roleEntities) { + roleEntitiesMap.put(entity.id(), entity); } - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); - List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); for (RoleEntity roleEntityToRevoke : roleEntitiesToRevoke) { - roleNames.remove(roleEntityToRevoke.name()); - boolean removed = roleIds.remove(roleEntityToRevoke.id()); - if (!removed) { + if (roleEntitiesMap.containsKey(roleEntityToRevoke.id())) { + roleEntitiesMap.remove(roleEntityToRevoke.id()); + } else { LOG.warn( "Failed to revoke, role {} does not exist in the group {} of metalake {}", roleEntityToRevoke.name(), @@ -261,8 +251,7 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) { .withNamespace(groupEntity.namespace()) .withId(groupEntity.id()) .withName(groupEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(Lists.newArrayList(roleEntitiesMap.values())) .withAuditInfo(auditInfo) .build(); }); @@ -308,20 +297,18 @@ User revokeRolesFromUser(String metalake, List roles, String user) { UserEntity.class, Entity.EntityType.USER, userEntity -> { - List roleEntities = Lists.newArrayList(); - if (userEntity.roleNames() != null) { - for (String role : userEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } - } + List roleEntities = userEntity.roleEntities(); - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); - List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + // Avoid loading securable objects + Map roleEntitiesMap = Maps.newHashMap(); + for (RoleEntity entity : roleEntities) { + roleEntitiesMap.put(entity.id(), entity); + } for (RoleEntity roleEntityToRevoke : roleEntitiesToRevoke) { - roleNames.remove(roleEntityToRevoke.name()); - boolean removed = roleIds.remove(roleEntityToRevoke.id()); - if (!removed) { + if (roleEntitiesMap.containsKey(roleEntityToRevoke.id())) { + roleEntitiesMap.remove(roleEntityToRevoke.id()); + } else { LOG.warn( "Failed to revoke, role {} doesn't exist in the user {} of metalake {}", roleEntityToRevoke.name(), @@ -341,8 +328,7 @@ User revokeRolesFromUser(String metalake, List roles, String user) { .withId(userEntity.id()) .withNamespace(userEntity.namespace()) .withName(userEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(Lists.newArrayList(roleEntitiesMap.values())) .withAuditInfo(auditInfo) .build(); }); @@ -373,10 +359,6 @@ User revokeRolesFromUser(String metalake, List roles, String user) { } } - private List toRoleNames(List roleEntities) { - return roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList()); - } - private List toRoleIds(List roleEntities) { return roleEntities.stream().map(RoleEntity::id).collect(Collectors.toList()); } diff --git a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java index 09427668969..eef947f22e3 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java @@ -63,7 +63,7 @@ User addUser(String metalake, String name) throws UserAlreadyExistsException { .withId(idGenerator.nextId()) .withName(name) .withNamespace(AuthorizationUtils.ofUserNamespace(metalake)) - .withRoleNames(Lists.newArrayList()) + .withRoles(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) @@ -117,7 +117,7 @@ Group addGroup(String metalake, String group) throws GroupAlreadyExistsException .withId(idGenerator.nextId()) .withName(group) .withNamespace(AuthorizationUtils.ofGroupNamespace(metalake)) - .withRoleNames(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) diff --git a/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java b/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java index d9e4b633d54..047c2bcce99 100644 --- a/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; import org.apache.gravitino.Field; @@ -38,11 +40,8 @@ public class GroupEntity implements Group, Entity, Auditable, HasIdentifier { public static final Field NAME = Field.required("name", String.class, "The name of the group entity."); - public static final Field ROLE_NAMES = - Field.optional("role_names", List.class, "The role names of the group entity."); - - public static final Field ROLE_IDS = - Field.optional("role_ids", List.class, "The role names of the group entity."); + public static final Field ROLES_SUPPLIER = + Field.required("roles_supplier", Supplier.class, "The roles supplier of the group entity."); public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the group entity."); @@ -50,9 +49,9 @@ public class GroupEntity implements Group, Entity, Auditable, HasIdentifier { private Long id; private String name; private AuditInfo auditInfo; - private List roleNames; - private List roleIds; private Namespace namespace; + // The roles is a lazy field to avoid unnecessary cost. + private Supplier> rolesSupplier = Collections::emptyList; private GroupEntity() {} @@ -67,8 +66,7 @@ public Map fields() { fields.put(ID, id); fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); - fields.put(ROLE_NAMES, roleNames); - fields.put(ROLE_IDS, roleIds); + fields.put(ROLES_SUPPLIER, rolesSupplier); return Collections.unmodifiableMap(fields); } @@ -130,25 +128,20 @@ public AuditInfo auditInfo() { */ @Override public List roles() { - return roleNames; - } + if (roleEntities() == null) { + return null; + } - /** - * Returns the role names of the group entity. - * - * @return The role names of the group entity. - */ - public List roleNames() { - return roleNames; + return roleEntities().stream().map(RoleEntity::name).collect(Collectors.toList()); } /** - * Returns the role ids of the group entity. + * Returns the role entities of the group entity. * - * @return The role ids of the group entity. + * @return The role entities of the group entity. */ - public List roleIds() { - return roleIds; + public List roleEntities() { + return rolesSupplier.get(); } @Override @@ -161,13 +154,12 @@ public boolean equals(Object o) { && Objects.equals(name, that.name) && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) - && Objects.equals(roleNames, that.roleNames) - && Objects.equals(roleIds, that.roleIds); + && Objects.equals(roles(), that.roles()); } @Override public int hashCode() { - return Objects.hash(id, name, auditInfo, roleNames, roleIds); + return Objects.hash(id, name, auditInfo, roles()); } public static Builder builder() { @@ -215,24 +207,26 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the role names of the group entity. + * Sets the roles of the group entity. * * @param roles The role names of the group entity. * @return The builder instance. */ - public Builder withRoleNames(List roles) { - groupEntity.roleNames = roles; + public Builder withRoles(List roles) { + if (roles != null) { + groupEntity.rolesSupplier = () -> roles; + } return this; } /** - * Sets the role ids of the group entity. + * Sets the roles supplier of the group entity. * - * @param roleIds The role ids of the group entity. + * @param supplier The roles supplier of the group entity. * @return The builder instance. */ - public Builder withRoleIds(List roleIds) { - groupEntity.roleIds = roleIds; + public Builder withRolesSupplier(Supplier> supplier) { + groupEntity.rolesSupplier = supplier; return this; } diff --git a/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java b/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java index d1b3bbfe99f..eda0bc89903 100644 --- a/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; import org.apache.gravitino.Field; @@ -46,15 +47,18 @@ public class RoleEntity implements Role, Entity, Auditable, HasIdentifier { public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the role entity."); - public static final Field SECURABLE_OBJECT = - Field.required("securable_objects", List.class, "The securable objects of the role entity."); + public static final Field SECURABLE_OBJECTS_SUPPLIER = + Field.required( + "securable_objects_supplier", + Supplier.class, + "The securable objects supplier of the role entity."); private Long id; private String name; private Map properties; private AuditInfo auditInfo; private Namespace namespace; - private List securableObjects; + private Supplier> securableObjectsSupplier = () -> null; /** * The name of the role. @@ -91,16 +95,7 @@ public Map properties() { */ @Override public List securableObjects() { - // The securable object is a special kind of entities. Some entity types aren't the securable - // object, such as - // User, Role, etc. - // The securable object identifier must be unique. - // Gravitino assumes that the identifiers of the entities may be the same if they have different - // types. - // So one type of them can't be the securable object at least if there are the two same - // identifier - // entities . - return securableObjects; + return securableObjectsSupplier.get(); } /** @@ -115,7 +110,7 @@ public Map fields() { fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); fields.put(PROPERTIES, properties); - fields.put(SECURABLE_OBJECT, securableObjects); + fields.put(SECURABLE_OBJECTS_SUPPLIER, securableObjectsSupplier); return Collections.unmodifiableMap(fields); } @@ -151,12 +146,12 @@ public boolean equals(Object o) { && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties) - && Objects.equals(securableObjects, that.securableObjects); + && Objects.equals(securableObjects(), that.securableObjects()); } @Override public int hashCode() { - return Objects.hash(id, name, properties, auditInfo, securableObjects, namespace); + return Objects.hash(id, name, properties, auditInfo, securableObjects(), namespace); } /** @@ -231,7 +226,19 @@ public Builder withAuditInfo(AuditInfo auditInfo) { * @return The builder instance. */ public Builder withSecurableObjects(List securableObjects) { - roleEntity.securableObjects = ImmutableList.copyOf(securableObjects); + roleEntity.securableObjectsSupplier = () -> ImmutableList.copyOf(securableObjects); + return this; + } + + /** + * Sets the securable objects supplier of the role entity. + * + * @param securableObjectsSupplier The securable objects supplier of the role entity. + * @return The builder instance. + */ + public Builder withSecurableObjectsSupplier( + Supplier> securableObjectsSupplier) { + roleEntity.securableObjectsSupplier = securableObjectsSupplier; return this; } diff --git a/core/src/main/java/org/apache/gravitino/meta/UserEntity.java b/core/src/main/java/org/apache/gravitino/meta/UserEntity.java index c71d731a99e..7e12bd2d64a 100644 --- a/core/src/main/java/org/apache/gravitino/meta/UserEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/UserEntity.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; import lombok.ToString; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; @@ -44,18 +46,15 @@ public class UserEntity implements User, Entity, Auditable, HasIdentifier { public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the user entity."); - public static final Field ROLE_NAMES = - Field.optional("role_names", List.class, "The role names of the user entity"); - - public static final Field ROLE_IDS = - Field.optional("role_ids", List.class, "The role ids of the user entity"); + public static final Field ROLES_SUPPLIER = + Field.required("roles_supplier", Supplier.class, "The roles supplier of the user entity"); private Long id; private String name; private AuditInfo auditInfo; - private List roleNames; - private List roleIds; private Namespace namespace; + // The roles is a lazy field to avoid unnecessary cost. + private Supplier> rolesSupplier = Collections::emptyList; private UserEntity() {} @@ -70,8 +69,7 @@ public Map fields() { fields.put(ID, id); fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); - fields.put(ROLE_NAMES, roleNames); - fields.put(ROLE_IDS, roleIds); + fields.put(ROLES_SUPPLIER, rolesSupplier); return Collections.unmodifiableMap(fields); } @@ -133,25 +131,16 @@ public AuditInfo auditInfo() { */ @Override public List roles() { - return roleNames; - } - - /** - * Returns the role names of the user entity. - * - * @return The role names of the user entity. - */ - public List roleNames() { - return roleNames; + return roleEntities().stream().map(RoleEntity::name).collect(Collectors.toList()); } /** - * Returns the role ids of the user entity. + * Returns the role entities of the user entity. * - * @return The role ids of the user entity. + * @return The role entities of the user entity. */ - public List roleIds() { - return roleIds; + public List roleEntities() { + return rolesSupplier.get(); } @Override @@ -164,13 +153,12 @@ public boolean equals(Object o) { && Objects.equals(name, that.name) && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) - && Objects.equals(roleNames, that.roleNames) - && Objects.equals(roleIds, that.roleIds); + && Objects.equals(roles(), that.roles()); } @Override public int hashCode() { - return Objects.hash(id, name, auditInfo, roleNames, roleIds); + return Objects.hash(id, name, auditInfo, roles()); } public static Builder builder() { @@ -218,24 +206,26 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the role names of the user entity. + * Sets the roles of the user entity. * - * @param roles The role names of the user entity. + * @param roles The role entities of the user entity. * @return The builder instance. */ - public Builder withRoleNames(List roles) { - userEntity.roleNames = roles; + public Builder withRoles(List roles) { + if (roles != null) { + userEntity.rolesSupplier = () -> roles; + } return this; } /** - * Sets the role ids of the user entity. + * Sets the roles supplier of the user entity. * - * @param roleIds The role ids of the user entity. + * @param supplier The roles supplier of the user entity. * @return The builder instance. */ - public Builder withRoleIds(List roleIds) { - userEntity.roleIds = roleIds; + public Builder withRolesSupplier(Supplier> supplier) { + userEntity.rolesSupplier = supplier; return this; } diff --git a/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java deleted file mode 100644 index 46eb9c90841..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import java.util.Collection; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.GroupEntity; - -public class GroupEntitySerDe implements ProtoSerDe { - - @Override - public Group serialize(GroupEntity groupEntity) { - Group.Builder builder = - Group.newBuilder() - .setId(groupEntity.id()) - .setName(groupEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(groupEntity.auditInfo())); - - if (isCollectionNotEmpty(groupEntity.roles())) { - builder.addAllRoleNames(groupEntity.roles()); - } - - if (isCollectionNotEmpty(groupEntity.roleIds())) { - builder.addAllRoleIds(groupEntity.roleIds()); - } - - return builder.build(); - } - - @Override - public GroupEntity deserialize(Group group, Namespace namespace) { - GroupEntity.Builder builder = - GroupEntity.builder() - .withId(group.getId()) - .withName(group.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo(), namespace)); - - if (group.getRoleNamesCount() > 0) { - builder.withRoleNames(group.getRoleNamesList()); - } - - if (group.getRoleIdsCount() > 0) { - builder.withRoleIds(group.getRoleIdsList()); - } - - return builder.build(); - } - - private boolean isCollectionNotEmpty(Collection collection) { - return collection != null && !collection.isEmpty(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java index 3dc5cb1c297..a4b01a896d8 100644 --- a/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java @@ -53,11 +53,6 @@ public class ProtoEntitySerDe implements EntitySerDe { .put( "org.apache.gravitino.meta.TopicEntity", "org.apache.gravitino.proto.TopicEntitySerDe") - .put("org.apache.gravitino.meta.UserEntity", "org.apache.gravitino.proto.UserEntitySerDe") - .put( - "org.apache.gravitino.meta.GroupEntity", - "org.apache.gravitino.proto.GroupEntitySerDe") - .put("org.apache.gravitino.meta.RoleEntity", "org.apache.gravitino.proto.RoleEntitySerDe") .build(); private static final Map ENTITY_TO_PROTO = @@ -75,13 +70,7 @@ public class ProtoEntitySerDe implements EntitySerDe { "org.apache.gravitino.meta.FilesetEntity", "org.apache.gravitino.proto.Fileset", "org.apache.gravitino.meta.TopicEntity", - "org.apache.gravitino.proto.Topic", - "org.apache.gravitino.meta.UserEntity", - "org.apache.gravitino.proto.User", - "org.apache.gravitino.meta.GroupEntity", - "org.apache.gravitino.proto.Group", - "org.apache.gravitino.meta.RoleEntity", - "org.apache.gravitino.proto.Role"); + "org.apache.gravitino.proto.Topic"); private final Map, ProtoSerDe> entityToSerDe; diff --git a/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java deleted file mode 100644 index a54f4505d7f..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import com.google.common.collect.Lists; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.gravitino.MetadataObject; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.authorization.Privilege; -import org.apache.gravitino.authorization.Privileges; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.meta.RoleEntity; - -public class RoleEntitySerDe implements ProtoSerDe { - - /** - * Serializes the provided entity into its corresponding Protocol Buffer message representation. - * - * @param roleEntity The entity to be serialized. - * @return The Protocol Buffer message representing the serialized entity. - */ - @Override - public Role serialize(RoleEntity roleEntity) { - Role.Builder builder = - Role.newBuilder() - .setId(roleEntity.id()) - .setName(roleEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())); - - for (SecurableObject securableObject : roleEntity.securableObjects()) { - builder.addSecurableObjects( - org.apache.gravitino.proto.SecurableObject.newBuilder() - .setFullName(securableObject.fullName()) - .setType(securableObject.type().name()) - .addAllPrivilegeConditions( - securableObject.privileges().stream() - .map(Privilege::condition) - .map(Privilege.Condition::name) - .collect(Collectors.toList())) - .addAllPrivilegeNames( - securableObject.privileges().stream() - .map(Privilege::name) - .map(Privilege.Name::name) - .collect(Collectors.toList())) - .build()); - } - - if (roleEntity.properties() != null && !roleEntity.properties().isEmpty()) { - builder.putAllProperties(roleEntity.properties()); - } - - return builder.build(); - } - - /** - * Deserializes the provided Protocol Buffer message into its corresponding entity representation. - * - * @param role The Protocol Buffer message to be deserialized. - * @return The entity representing the deserialized Protocol Buffer message. - */ - @Override - public RoleEntity deserialize(Role role, Namespace namespace) { - List securableObjects = Lists.newArrayList(); - - for (int index = 0; index < role.getSecurableObjectsCount(); index++) { - List privileges = Lists.newArrayList(); - org.apache.gravitino.proto.SecurableObject object = role.getSecurableObjects(index); - for (int privIndex = 0; privIndex < object.getPrivilegeConditionsCount(); privIndex++) { - if (Privilege.Condition.ALLOW.name().equals(object.getPrivilegeConditions(privIndex))) { - privileges.add(Privileges.allow(object.getPrivilegeNames(privIndex))); - } else { - privileges.add(Privileges.deny(object.getPrivilegeNames(privIndex))); - } - } - - SecurableObject securableObject = - SecurableObjects.parse( - object.getFullName(), MetadataObject.Type.valueOf(object.getType()), privileges); - - securableObjects.add(securableObject); - } - - RoleEntity.Builder builder = - RoleEntity.builder() - .withId(role.getId()) - .withName(role.getName()) - .withNamespace(namespace) - .withSecurableObjects(securableObjects) - .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); - - if (!role.getPropertiesMap().isEmpty()) { - builder.withProperties(role.getPropertiesMap()); - } - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java deleted file mode 100644 index e859ee3b881..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import java.util.Collection; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.UserEntity; - -public class UserEntitySerDe implements ProtoSerDe { - - @Override - public User serialize(UserEntity userEntity) { - User.Builder builder = - User.newBuilder() - .setId(userEntity.id()) - .setName(userEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(userEntity.auditInfo())); - - if (isCollectionNotEmpty(userEntity.roles())) { - builder.addAllRoleNames(userEntity.roles()); - } - - if (isCollectionNotEmpty(userEntity.roleIds())) { - builder.addAllRoleIds(userEntity.roleIds()); - } - - return builder.build(); - } - - @Override - public UserEntity deserialize(User user, Namespace namespace) { - UserEntity.Builder builder = - UserEntity.builder() - .withId(user.getId()) - .withName(user.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo(), namespace)); - - if (user.getRoleNamesCount() > 0) { - builder.withRoleNames(user.getRoleNamesList()); - } - - if (user.getRoleIdsCount() > 0) { - builder.withRoleIds(user.getRoleIdsList()); - } - - return builder.build(); - } - - private boolean isCollectionNotEmpty(Collection collection) { - return collection != null && !collection.isEmpty(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java b/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java index fa6fb48c636..2dd0e00b0cf 100644 --- a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java +++ b/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java @@ -20,13 +20,10 @@ import static org.apache.gravitino.Entity.EntityType.CATALOG; import static org.apache.gravitino.Entity.EntityType.FILESET; -import static org.apache.gravitino.Entity.EntityType.GROUP; import static org.apache.gravitino.Entity.EntityType.METALAKE; -import static org.apache.gravitino.Entity.EntityType.ROLE; import static org.apache.gravitino.Entity.EntityType.SCHEMA; import static org.apache.gravitino.Entity.EntityType.TABLE; import static org.apache.gravitino.Entity.EntityType.TOPIC; -import static org.apache.gravitino.Entity.EntityType.USER; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -93,12 +90,6 @@ public class BinaryEntityKeyEncoder implements EntityKeyEncoder { new String[] {TABLE.getShortName() + "/", "/", "/", "/"}, FILESET, new String[] {FILESET.getShortName() + "/", "/", "/", "/"}, - USER, - new String[] {USER.getShortName() + "/", "/", "/", "/"}, - GROUP, - new String[] {GROUP.getShortName() + "/", "/", "/", "/"}, - ROLE, - new String[] {ROLE.getShortName() + "/", "/", "/", "/"}, TOPIC, new String[] {TOPIC.getShortName() + "/", "/", "/", "/"}); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java index 2ffc10dac59..2762d55a7fd 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java @@ -22,10 +22,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -41,10 +39,10 @@ import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.po.GroupPO; import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; -import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; /** The service class for group metadata. It provides the basic database operations for group. */ public class GroupMetaService { @@ -92,9 +90,9 @@ public GroupEntity getGroupByIdentifier(NameIdentifier identifier) { Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); GroupPO groupPO = getGroupPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = RoleMetaService.getInstance().listRolesByGroupId(groupPO.getGroupId()); - return POConverters.fromGroupPO(groupPO, rolePOs, identifier.namespace()); + return POConverters.fromGroupPO( + groupPO, SupplierUtils.createRolePOsSupplier(groupPO), identifier.namespace()); } public List listGroupsByRoleIdent(NameIdentifier roleIdent) { @@ -107,7 +105,7 @@ public List listGroupsByRoleIdent(NameIdentifier roleIdent) { po -> POConverters.fromGroupPO( po, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(po), AuthorizationUtils.ofGroupNamespace(roleIdent.namespace().level(0)))) .collect(Collectors.toList()); } @@ -121,7 +119,8 @@ public void insertGroup(GroupEntity groupEntity, boolean overwritten) throws IOE GroupPO.Builder builder = GroupPO.builder().withMetalakeId(metalakeId); GroupPO GroupPO = POConverters.initializeGroupPOWithVersion(groupEntity, builder); - List roleIds = Optional.ofNullable(groupEntity.roleIds()).orElse(Lists.newArrayList()); + List roleIds = + groupEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toList()); List groupRoleRelPOS = POConverters.initializeGroupRoleRelsPOWithVersion(groupEntity, roleIds); @@ -186,10 +185,10 @@ public GroupEntity updateGroup( Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); GroupPO oldGroupPO = getGroupPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = - RoleMetaService.getInstance().listRolesByGroupId(oldGroupPO.getGroupId()); + GroupEntity oldGroupEntity = - POConverters.fromGroupPO(oldGroupPO, rolePOs, identifier.namespace()); + POConverters.fromGroupPO( + oldGroupPO, SupplierUtils.createRolePOsSupplier(oldGroupPO), identifier.namespace()); GroupEntity newEntity = (GroupEntity) updater.apply((E) oldGroupEntity); Preconditions.checkArgument( @@ -199,11 +198,10 @@ public GroupEntity updateGroup( oldGroupEntity.id()); Set oldRoleIds = - oldGroupEntity.roleIds() == null - ? Sets.newHashSet() - : Sets.newHashSet(oldGroupEntity.roleIds()); + oldGroupEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); + Set newRoleIds = - newEntity.roleIds() == null ? Sets.newHashSet() : Sets.newHashSet(newEntity.roleIds()); + newEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); Set insertRoleIds = Sets.difference(newRoleIds, oldRoleIds); Set deleteRoleIds = Sets.difference(oldRoleIds, newRoleIds); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java index 1118467b0a7..502b427f045 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.storage.relational.service; -import java.util.Collections; import java.util.Optional; import org.apache.gravitino.Entity; import org.apache.gravitino.MetadataObject; @@ -30,6 +29,7 @@ import org.apache.gravitino.storage.relational.po.UserPO; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; import org.apache.gravitino.utils.NameIdentifierUtil; /** This class is an utilization class to retrieve owner relation. */ @@ -58,7 +58,7 @@ public Optional getOwner(NameIdentifier identifier, Entity.EntityType ty return Optional.of( POConverters.fromUserPO( userPO, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(userPO), AuthorizationUtils.ofUserNamespace(NameIdentifierUtil.getMetalake(identifier)))); } @@ -71,7 +71,7 @@ public Optional getOwner(NameIdentifier identifier, Entity.EntityType ty return Optional.of( POConverters.fromGroupPO( groupPO, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(groupPO), AuthorizationUtils.ofGroupNamespace(NameIdentifierUtil.getMetalake(identifier)))); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 4560b74e0d0..743eb6338d2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -39,14 +39,12 @@ import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; import org.apache.gravitino.utils.NameIdentifierUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** The service class for role metadata. It provides the basic database operations for role. */ public class RoleMetaService { - private static final Logger LOG = LoggerFactory.getLogger(RoleMetaService.class); private static final RoleMetaService INSTANCE = new RoleMetaService(); public static RoleMetaService getInstance() { @@ -109,33 +107,12 @@ public List listRolesByMetadataObjectIdentAndType( .map( po -> POConverters.fromRolePO( - po, listSecurableObjects(po), AuthorizationUtils.ofRoleNamespace(metalake))) + po, + SupplierUtils.createSecurableObjectsSupplier(po), + AuthorizationUtils.ofRoleNamespace(metalake))) .collect(Collectors.toList()); } - private List listSecurableObjects(RolePO po) { - List securableObjectPOs = listSecurableObjectsByRoleId(po.getRoleId()); - List securableObjects = Lists.newArrayList(); - - for (SecurableObjectPO securableObjectPO : securableObjectPOs) { - String fullName = - MetadataObjectService.getMetadataObjectFullName( - securableObjectPO.getType(), securableObjectPO.getMetadataObjectId()); - if (fullName != null) { - securableObjects.add( - POConverters.fromSecurableObjectPO( - fullName, securableObjectPO, getType(securableObjectPO.getType()))); - } else { - LOG.info( - "The securable object {} {} may be deleted", - securableObjectPO.getMetadataObjectId(), - securableObjectPO.getType()); - } - } - - return securableObjects; - } - public List listRolesByGroupId(Long groupId) { return SessionUtils.getWithoutCommit( RoleMetaMapper.class, mapper -> mapper.listRolesByGroupId(groupId)); @@ -197,9 +174,8 @@ public RoleEntity getRoleByIdentifier(NameIdentifier identifier) { MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); RolePO rolePO = getRolePOByMetalakeIdAndName(metalakeId, identifier.name()); - List securableObjects = listSecurableObjects(rolePO); - - return POConverters.fromRolePO(rolePO, securableObjects, identifier.namespace()); + return POConverters.fromRolePO( + rolePO, SupplierUtils.createSecurableObjectsSupplier(rolePO), identifier.namespace()); } public boolean deleteRole(NameIdentifier identifier) { @@ -232,7 +208,7 @@ public boolean deleteRole(NameIdentifier identifier) { return true; } - private List listSecurableObjectsByRoleId(Long roleId) { + public List listSecurableObjectsByRoleId(Long roleId) { return SessionUtils.getWithoutCommit( SecurableObjectMapper.class, mapper -> mapper.listSecurableObjectsByRoleId(roleId)); } @@ -273,10 +249,6 @@ public int deleteRoleMetasByLegacyTimeline(long legacyTimeline, int limit) { + securableObjectsCount[0]; } - private MetadataObject.Type getType(String type) { - return MetadataObject.Type.valueOf(type); - } - private String getEntityType(SecurableObject securableObject) { return securableObject.type().name(); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java index e7d0a435a1b..67bbfbce3df 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java @@ -22,10 +22,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -39,12 +37,12 @@ import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.mapper.UserMetaMapper; import org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper; -import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.po.UserPO; import org.apache.gravitino.storage.relational.po.UserRoleRelPO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; /** The service class for user metadata. It provides the basic database operations for user. */ public class UserMetaService { @@ -92,9 +90,9 @@ public UserEntity getUserByIdentifier(NameIdentifier identifier) { Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); UserPO userPO = getUserPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = RoleMetaService.getInstance().listRolesByUserId(userPO.getUserId()); - return POConverters.fromUserPO(userPO, rolePOs, identifier.namespace()); + return POConverters.fromUserPO( + userPO, SupplierUtils.createRolePOsSupplier(userPO), identifier.namespace()); } public List listUsersByRoleIdent(NameIdentifier roleIdent) { @@ -107,7 +105,7 @@ public List listUsersByRoleIdent(NameIdentifier roleIdent) { po -> POConverters.fromUserPO( po, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(po), AuthorizationUtils.ofUserNamespace(roleIdent.namespace().level(0)))) .collect(Collectors.toList()); } @@ -121,7 +119,8 @@ public void insertUser(UserEntity userEntity, boolean overwritten) throws IOExce UserPO.Builder builder = UserPO.builder().withMetalakeId(metalakeId); UserPO userPO = POConverters.initializeUserPOWithVersion(userEntity, builder); - List roleIds = Optional.ofNullable(userEntity.roleIds()).orElse(Lists.newArrayList()); + List roleIds = + userEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toList()); List userRoleRelPOs = POConverters.initializeUserRoleRelsPOWithVersion(userEntity, roleIds); @@ -185,8 +184,9 @@ public UserEntity updateUser( Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); UserPO oldUserPO = getUserPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = RoleMetaService.getInstance().listRolesByUserId(oldUserPO.getUserId()); - UserEntity oldUserEntity = POConverters.fromUserPO(oldUserPO, rolePOs, identifier.namespace()); + UserEntity oldUserEntity = + POConverters.fromUserPO( + oldUserPO, SupplierUtils.createRolePOsSupplier(oldUserPO), identifier.namespace()); UserEntity newEntity = (UserEntity) updater.apply((E) oldUserEntity); Preconditions.checkArgument( @@ -196,11 +196,10 @@ public UserEntity updateUser( oldUserEntity.id()); Set oldRoleIds = - oldUserEntity.roleIds() == null - ? Sets.newHashSet() - : Sets.newHashSet(oldUserEntity.roleIds()); + oldUserEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); + Set newRoleIds = - newEntity.roleIds() == null ? Sets.newHashSet() : Sets.newHashSet(newEntity.roleIds()); + newEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); Set insertRoleIds = Sets.difference(newRoleIds, oldRoleIds); Set deleteRoleIds = Sets.difference(oldRoleIds, newRoleIds); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java index 82d739a41cc..c21ef1df08e 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java @@ -24,11 +24,13 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.Namespace; +import org.apache.gravitino.authorization.AuthorizationUtils; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; @@ -699,29 +701,31 @@ public static UserPO updateUserPOWithVersion(UserPO oldUserPO, UserEntity newUse * Convert {@link UserPO} to {@link UserEntity} * * @param userPO UserPo object to be converted - * @param rolePOs list of rolePO + * @param rolePOsSupplier Supplier for the list of rolePO * @param namespace Namespace object to be associated with the user * @return UserEntity object from UserPO object */ - public static UserEntity fromUserPO(UserPO userPO, List rolePOs, Namespace namespace) { + public static UserEntity fromUserPO( + UserPO userPO, Supplier> rolePOsSupplier, Namespace namespace) { try { - List roleNames = - rolePOs.stream().map(RolePO::getRoleName).collect(Collectors.toList()); - List roleIds = rolePOs.stream().map(RolePO::getRoleId).collect(Collectors.toList()); - UserEntity.Builder builder = UserEntity.builder() .withId(userPO.getUserId()) .withName(userPO.getUserName()) .withNamespace(namespace) .withAuditInfo( - JsonUtils.anyFieldMapper().readValue(userPO.getAuditInfo(), AuditInfo.class)); - if (!roleNames.isEmpty()) { - builder.withRoleNames(roleNames); - } - if (!roleIds.isEmpty()) { - builder.withRoleIds(roleIds); - } + JsonUtils.anyFieldMapper().readValue(userPO.getAuditInfo(), AuditInfo.class)) + .withRolesSupplier( + () -> + rolePOsSupplier.get().stream() + .map( + po -> + fromRolePO( + po, + SupplierUtils.createSecurableObjectsSupplier(po), + AuthorizationUtils.ofRoleNamespace(namespace.level(0)))) + .collect(Collectors.toList())); + return builder.build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to deserialize json object:", e); @@ -732,30 +736,31 @@ public static UserEntity fromUserPO(UserPO userPO, List rolePOs, Namespa * Convert {@link GroupPO} to {@link GroupEntity} * * @param groupPO GroupPO object to be converted - * @param rolePOs list of rolePO + * @param rolePOsSupplier Supplier for the list of rolePO * @param namespace Namespace object to be associated with the group * @return GroupEntity object from GroupPO object */ public static GroupEntity fromGroupPO( - GroupPO groupPO, List rolePOs, Namespace namespace) { + GroupPO groupPO, Supplier> rolePOsSupplier, Namespace namespace) { try { - List roleNames = - rolePOs.stream().map(RolePO::getRoleName).collect(Collectors.toList()); - List roleIds = rolePOs.stream().map(RolePO::getRoleId).collect(Collectors.toList()); - GroupEntity.Builder builder = GroupEntity.builder() .withId(groupPO.getGroupId()) .withName(groupPO.getGroupName()) .withNamespace(namespace) .withAuditInfo( - JsonUtils.anyFieldMapper().readValue(groupPO.getAuditInfo(), AuditInfo.class)); - if (!roleNames.isEmpty()) { - builder.withRoleNames(roleNames); - } - if (!roleIds.isEmpty()) { - builder.withRoleIds(roleIds); - } + JsonUtils.anyFieldMapper().readValue(groupPO.getAuditInfo(), AuditInfo.class)) + .withRolesSupplier( + () -> + rolePOsSupplier.get().stream() + .map( + po -> + fromRolePO( + po, + SupplierUtils.createSecurableObjectsSupplier(po), + AuthorizationUtils.ofRoleNamespace(namespace.level(0)))) + .collect(Collectors.toList())); + return builder.build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to deserialize json object:", e); @@ -918,14 +923,16 @@ public static SecurableObject fromSecurableObjectPO( } public static RoleEntity fromRolePO( - RolePO rolePO, List securableObjects, Namespace namespace) { + RolePO rolePO, + Supplier> securableObjectsSupplier, + Namespace namespace) { try { return RoleEntity.builder() .withId(rolePO.getRoleId()) .withName(rolePO.getRoleName()) .withNamespace(namespace) .withProperties(JsonUtils.anyFieldMapper().readValue(rolePO.getProperties(), Map.class)) - .withSecurableObjects(securableObjects) + .withSecurableObjectsSupplier(securableObjectsSupplier) .withAuditInfo( JsonUtils.anyFieldMapper().readValue(rolePO.getAuditInfo(), AuditInfo.class)) .build(); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/utils/SupplierUtils.java b/core/src/main/java/org/apache/gravitino/storage/relational/utils/SupplierUtils.java new file mode 100644 index 00000000000..16c662c476d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/utils/SupplierUtils.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.utils; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.function.Supplier; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.storage.relational.po.GroupPO; +import org.apache.gravitino.storage.relational.po.RolePO; +import org.apache.gravitino.storage.relational.po.SecurableObjectPO; +import org.apache.gravitino.storage.relational.po.UserPO; +import org.apache.gravitino.storage.relational.service.MetadataObjectService; +import org.apache.gravitino.storage.relational.service.RoleMetaService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* This class is a utilization class for creating kinds of suppliers */ +public class SupplierUtils { + + private static final Logger LOG = LoggerFactory.getLogger(SupplierUtils.class); + + private SupplierUtils() {} + + public static Supplier> createRolePOsSupplier(UserPO userPO) { + return new Supplier>() { + private List rolePOs; + private boolean waitToLoad = true; + + @Override + public List get() { + if (waitToLoad) { + rolePOs = RoleMetaService.getInstance().listRolesByUserId(userPO.getUserId()); + waitToLoad = false; + } + + return rolePOs; + } + }; + } + + public static Supplier> createRolePOsSupplier(GroupPO groupPO) { + return new Supplier>() { + private List rolePOS; + private boolean waitToLoad = true; + + @Override + public List get() { + if (waitToLoad) { + rolePOS = RoleMetaService.getInstance().listRolesByGroupId(groupPO.getGroupId()); + waitToLoad = false; + } + + return rolePOS; + } + }; + } + + public static Supplier> createSecurableObjectsSupplier(RolePO rolePO) { + return new Supplier>() { + private List securableObjects; + + private boolean waitToLoad = true; + + @Override + public List get() { + if (waitToLoad) { + List securableObjectPOs = + RoleMetaService.getInstance().listSecurableObjectsByRoleId(rolePO.getRoleId()); + + securableObjects = Lists.newArrayList(); + + for (SecurableObjectPO securableObjectPO : securableObjectPOs) { + String fullName = + MetadataObjectService.getMetadataObjectFullName( + securableObjectPO.getType(), securableObjectPO.getMetadataObjectId()); + if (fullName != null) { + securableObjects.add( + POConverters.fromSecurableObjectPO( + fullName, + securableObjectPO, + MetadataObject.Type.valueOf(securableObjectPO.getType()))); + } else { + LOG.info( + "The securable object {} {} may be deleted", + securableObjectPO.getMetadataObjectId(), + securableObjectPO.getType()); + } + } + waitToLoad = false; + } + + return securableObjects; + } + }; + } +} diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java index ce3c4e90c9d..946cebf972e 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -157,7 +157,7 @@ public void testGrantRoleToUser() { String notExist = "not-exist"; User user = accessControlManager.getUser(METALAKE, USER); - Assertions.assertNull(user.roles()); + Assertions.assertTrue(user.roles().isEmpty()); reset(authorizationPlugin); diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java b/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java index 83a562f640d..0b0b93ef056 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java @@ -125,8 +125,7 @@ public static void setUp() throws IOException, IllegalAccessException { UserEntity.builder() .withId(idGenerator.nextId()) .withName(USER) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withNamespace(AuthorizationUtils.ofUserNamespace(METALAKE)) .withAuditInfo(audit) .build(); @@ -136,8 +135,7 @@ public static void setUp() throws IOException, IllegalAccessException { GroupEntity.builder() .withId(idGenerator.nextId()) .withName(GROUP) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withNamespace(AuthorizationUtils.ofUserNamespace(METALAKE)) .withAuditInfo(audit) .build(); diff --git a/core/src/test/java/org/apache/gravitino/meta/TestEntity.java b/core/src/test/java/org/apache/gravitino/meta/TestEntity.java index f4c32dca718..2e24a7eb013 100644 --- a/core/src/test/java/org/apache/gravitino/meta/TestEntity.java +++ b/core/src/test/java/org/apache/gravitino/meta/TestEntity.java @@ -21,10 +21,15 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.time.Instant; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import org.apache.gravitino.Catalog; import org.apache.gravitino.Field; import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.file.Fileset; import org.junit.jupiter.api.Assertions; @@ -237,19 +242,48 @@ public void testUser() { .withId(userId) .withName(userName) .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) + .withRoles( + Lists.newArrayList( + RoleEntity.builder() + .withId(1L) + .withName("role") + .withAuditInfo(auditInfo) + .build())) .build(); Map fields = testUserEntity.fields(); Assertions.assertEquals(userId, fields.get(UserEntity.ID)); Assertions.assertEquals(userName, fields.get(UserEntity.NAME)); Assertions.assertEquals(auditInfo, fields.get(UserEntity.AUDIT_INFO)); - Assertions.assertEquals(Lists.newArrayList("role"), fields.get(UserEntity.ROLE_NAMES)); + Assertions.assertEquals( + Lists.newArrayList( + RoleEntity.builder().withId(1L).withName("role").withAuditInfo(auditInfo).build()), + ((Supplier>) fields.get(UserEntity.ROLES_SUPPLIER)).get()); UserEntity testUserEntityWithoutFields = UserEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); - Assertions.assertNull(testUserEntityWithoutFields.roles()); + Assertions.assertTrue(testUserEntityWithoutFields.roles().isEmpty()); + + // Test lazily loading + AtomicBoolean hasCallRolesSupplier = new AtomicBoolean(false); + + UserEntity userWithSupplier = + UserEntity.builder() + .withId(userId) + .withName(userName) + .withAuditInfo(auditInfo) + .withRolesSupplier( + () -> { + hasCallRolesSupplier.set(true); + return Collections.emptyList(); + }) + .build(); + + Assertions.assertFalse(hasCallRolesSupplier.get()); + + userWithSupplier.roles(); + Assertions.assertTrue(hasCallRolesSupplier.get()); } @Test @@ -258,19 +292,48 @@ public void testGroup() { GroupEntity.builder() .withId(groupId) .withName(groupName) + .withRoles( + Lists.newArrayList( + RoleEntity.builder() + .withId(1L) + .withName("role") + .withAuditInfo(auditInfo) + .build())) .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) .build(); Map fields = group.fields(); Assertions.assertEquals(groupId, fields.get(GroupEntity.ID)); Assertions.assertEquals(groupName, fields.get(GroupEntity.NAME)); Assertions.assertEquals(auditInfo, fields.get(GroupEntity.AUDIT_INFO)); - Assertions.assertEquals(Lists.newArrayList("role"), fields.get(GroupEntity.ROLE_NAMES)); + Assertions.assertEquals( + Lists.newArrayList( + RoleEntity.builder().withId(1L).withName("role").withAuditInfo(auditInfo).build()), + ((Supplier>) fields.get(GroupEntity.ROLES_SUPPLIER)).get()); GroupEntity groupWithoutFields = GroupEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); - Assertions.assertNull(groupWithoutFields.roles()); + Assertions.assertTrue(groupWithoutFields.roles().isEmpty()); + + AtomicBoolean hasCallRolesSupplier = new AtomicBoolean(false); + + // Test lazily loading + GroupEntity groupWithSupplier = + GroupEntity.builder() + .withId(userId) + .withName(userName) + .withAuditInfo(auditInfo) + .withRolesSupplier( + () -> { + hasCallRolesSupplier.set(true); + return Collections.emptyList(); + }) + .build(); + + Assertions.assertFalse(hasCallRolesSupplier.get()); + + groupWithSupplier.roles(); + Assertions.assertTrue(hasCallRolesSupplier.get()); } @Test @@ -296,7 +359,8 @@ public void testRole() { Lists.newArrayList( SecurableObjects.ofCatalog( catalogName, Lists.newArrayList(Privileges.UseCatalog.allow()))), - fields.get(RoleEntity.SECURABLE_OBJECT)); + ((Supplier>) fields.get(RoleEntity.SECURABLE_OBJECTS_SUPPLIER)) + .get()); RoleEntity roleWithoutFields = RoleEntity.builder() @@ -309,6 +373,26 @@ public void testRole() { catalogName, Lists.newArrayList(Privileges.UseCatalog.allow())))) .build(); Assertions.assertNull(roleWithoutFields.properties()); + + // Test lazily loading + AtomicBoolean hasCallSecurableObjectsSupplier = new AtomicBoolean(false); + + RoleEntity roleWithSupplier = + RoleEntity.builder() + .withId(userId) + .withName(userName) + .withAuditInfo(auditInfo) + .withSecurableObjectsSupplier( + () -> { + hasCallSecurableObjectsSupplier.set(true); + return null; + }) + .build(); + + Assertions.assertFalse(hasCallSecurableObjectsSupplier.get()); + + roleWithSupplier.securableObjects(); + Assertions.assertTrue(hasCallSecurableObjectsSupplier.get()); } @Test diff --git a/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java index 1d4bfaa17d9..226199a0d93 100644 --- a/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java @@ -19,23 +19,15 @@ package org.apache.gravitino.proto; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import java.io.IOException; import java.time.Instant; import java.util.Map; import org.apache.gravitino.Catalog; -import org.apache.gravitino.Entity; import org.apache.gravitino.EntitySerDe; import org.apache.gravitino.EntitySerDeFactory; import org.apache.gravitino.Namespace; -import org.apache.gravitino.authorization.Privileges; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.meta.GroupEntity; -import org.apache.gravitino.meta.RoleEntity; import org.apache.gravitino.meta.SchemaVersion; -import org.apache.gravitino.meta.UserEntity; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -323,110 +315,5 @@ public void testEntitiesSerDe() throws IOException { Assertions.assertEquals(topicEntity1, topicEntityFromBytes1); Assertions.assertNull(topicEntityFromBytes1.comment()); Assertions.assertNull(topicEntityFromBytes1.properties()); - - // Test UserEntity - Namespace userNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME); - Long userId = 1L; - String userName = "user"; - UserEntity userEntity = - UserEntity.builder() - .withId(userId) - .withName(userName) - .withNamespace(userNamespace) - .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) - .withRoleIds(Lists.newArrayList(1L)) - .build(); - byte[] userBytes = protoEntitySerDe.serialize(userEntity); - UserEntity userEntityFromBytes = - protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); - Assertions.assertEquals(userEntity, userEntityFromBytes); - - UserEntity userEntityWithoutFields = - UserEntity.builder() - .withId(userId) - .withName(userName) - .withNamespace(userNamespace) - .withAuditInfo(auditInfo) - .build(); - userBytes = protoEntitySerDe.serialize(userEntityWithoutFields); - userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); - Assertions.assertEquals(userEntityWithoutFields, userEntityFromBytes); - Assertions.assertNull(userEntityWithoutFields.roles()); - Assertions.assertNull(userEntityWithoutFields.roleIds()); - - // Test GroupEntity - Namespace groupNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME); - Long groupId = 1L; - String groupName = "group"; - - GroupEntity group = - GroupEntity.builder() - .withId(groupId) - .withName(groupName) - .withNamespace(groupNamespace) - .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) - .withRoleIds(Lists.newArrayList(1L)) - .build(); - byte[] groupBytes = protoEntitySerDe.serialize(group); - GroupEntity groupFromBytes = - protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); - Assertions.assertEquals(group, groupFromBytes); - - GroupEntity groupWithoutFields = - GroupEntity.builder() - .withId(groupId) - .withName(groupName) - .withNamespace(groupNamespace) - .withAuditInfo(auditInfo) - .build(); - groupBytes = protoEntitySerDe.serialize(groupWithoutFields); - groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); - Assertions.assertEquals(groupWithoutFields, groupFromBytes); - Assertions.assertNull(groupWithoutFields.roles()); - Assertions.assertNull(groupWithoutFields.roleIds()); - - // Test RoleEntity - Namespace roleNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME); - Long roleId = 1L; - String roleName = "testRole"; - String anotherCatalogName = "anotherCatalog"; - SecurableObject securableObject = - SecurableObjects.ofCatalog( - catalogName, - Lists.newArrayList(Privileges.UseCatalog.allow(), Privileges.CreateSchema.deny())); - SecurableObject anotherSecurableObject = - SecurableObjects.ofCatalog( - anotherCatalogName, Lists.newArrayList(Privileges.UseCatalog.allow())); - - RoleEntity roleEntity = - RoleEntity.builder() - .withId(roleId) - .withName(roleName) - .withNamespace(roleNamespace) - .withAuditInfo(auditInfo) - .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) - .withProperties(props) - .build(); - byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); - RoleEntity roleFromBytes = - protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); - Assertions.assertEquals(roleEntity, roleFromBytes); - - RoleEntity roleWithoutFields = - RoleEntity.builder() - .withId(1L) - .withName(roleName) - .withNamespace(roleNamespace) - .withAuditInfo(auditInfo) - .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) - .build(); - roleBytes = protoEntitySerDe.serialize(roleWithoutFields); - roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); - Assertions.assertEquals(roleWithoutFields, roleFromBytes); } } diff --git a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java index bdf95ff9639..d233ac5bafc 100644 --- a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java @@ -46,6 +46,7 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.UUID; import org.apache.commons.io.FileUtils; @@ -1234,7 +1235,7 @@ private static UserEntity createUser(Long id, String metalake, String name, Audi .withNamespace(AuthorizationUtils.ofUserNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); } @@ -1245,7 +1246,7 @@ private static GroupEntity createGroup( .withNamespace(AuthorizationUtils.ofGroupNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); } diff --git a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java index 7cff7bcc553..4d41d2f2523 100644 --- a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.locks.Lock; @@ -224,7 +225,7 @@ public void testEntityStoreAndRetrieve() throws Exception { .withName("user") .withNamespace(Namespace.of("metalake", "catalog", "db")) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); GroupEntity groupEntity = @@ -233,7 +234,7 @@ public void testEntityStoreAndRetrieve() throws Exception { .withName("group") .withNamespace(AuthorizationUtils.ofGroupNamespace("metalake")) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); RoleEntity roleEntity = diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java index 26430d2fb5e..d72f542fa1f 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java @@ -45,6 +45,7 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -600,8 +601,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofUserNamespace("metalake"), "user", auditInfo, - Lists.newArrayList(role.name()), - Lists.newArrayList(role.id())); + Lists.newArrayList(role)); backend.insert(user, false); GroupEntity group = @@ -610,8 +610,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofGroupNamespace("metalake"), "group", auditInfo, - Lists.newArrayList(role.name()), - Lists.newArrayList(role.id())); + Lists.newArrayList(role)); backend.insert(group, false); TagEntity tag = @@ -688,8 +687,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofUserNamespace("another-metalake"), "another-user", auditInfo, - Lists.newArrayList(anotherRole.name()), - Lists.newArrayList(anotherRole.id())); + Lists.newArrayList(anotherRole)); backend.insert(anotherUser, false); GroupEntity anotherGroup = @@ -698,8 +696,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofGroupNamespace("another-metalake"), "another-group", auditInfo, - Lists.newArrayList(anotherRole.name()), - Lists.newArrayList(anotherRole.id())); + Lists.newArrayList(anotherRole)); backend.insert(anotherGroup, false); TagEntity anotherTagEntity = @@ -1112,8 +1109,7 @@ public static UserEntity createUserEntity( .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(null) - .withRoleIds(null) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); } @@ -1123,14 +1119,12 @@ public static UserEntity createUserEntity( Namespace namespace, String name, AuditInfo auditInfo, - List roleNames, - List roleIds) { + List roleEntities) { return UserEntity.builder() .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(auditInfo) .build(); } @@ -1156,25 +1150,18 @@ public static GroupEntity createGroupEntity( .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(null) - .withRoleIds(null) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); } public static GroupEntity createGroupEntity( - Long id, - Namespace namespace, - String name, - AuditInfo auditInfo, - List roleNames, - List roleIds) { + Long id, Namespace namespace, String name, AuditInfo auditInfo, List roles) { return GroupEntity.builder() .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roles) .withAuditInfo(auditInfo) .build(); } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java index 22246ba0cf3..d6b33095045 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.Namespace; @@ -110,13 +111,11 @@ void getGroupByIdentifier() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group2, false); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); Assertions.assertEquals(group2.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group2.roles()), Sets.newHashSet(actualGroup.roles())); } @Test @@ -194,13 +193,11 @@ void insertGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group2, false)); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); Assertions.assertEquals(group2.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group2.roles()), Sets.newHashSet(actualGroup.roles())); // insert duplicate group with roles GroupEntity group2Exist = @@ -219,17 +216,16 @@ void insertGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2Overwrite", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group2Overwrite, true)); GroupEntity actualOverwriteGroup2 = groupMetaService.getGroupByIdentifier(group2Overwrite.nameIdentifier()); Assertions.assertEquals("group2Overwrite", actualOverwriteGroup2.name()); - Assertions.assertEquals(2, actualOverwriteGroup2.roleNames().size()); + Assertions.assertEquals(2, actualOverwriteGroup2.roles().size()); Assertions.assertEquals( Sets.newHashSet(role1.name(), role2.name()), - Sets.newHashSet(actualOverwriteGroup2.roleNames())); + Sets.newHashSet(actualOverwriteGroup2.roles())); // insert overwrite user with 1 role RoleEntity role3 = @@ -246,15 +242,14 @@ void insertGroup() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "group3Overwrite", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group3Overwrite, true)); GroupEntity actualOverwriteGroup3 = groupMetaService.getGroupByIdentifier(group3Overwrite.nameIdentifier()); Assertions.assertEquals("group3Overwrite", actualOverwriteGroup3.name()); - Assertions.assertEquals(1, actualOverwriteGroup3.roleNames().size()); - Assertions.assertEquals("role3", actualOverwriteGroup3.roleNames().get(0)); + Assertions.assertEquals(1, actualOverwriteGroup3.roles().size()); + Assertions.assertEquals("role3", actualOverwriteGroup3.roles().get(0)); // insert overwrite user with 0 roles GroupEntity group4Overwrite = @@ -268,7 +263,7 @@ void insertGroup() throws IOException { GroupEntity actualOverwriteGroup4 = groupMetaService.getGroupByIdentifier(group4Overwrite.nameIdentifier()); Assertions.assertEquals("group4Overwrite", actualOverwriteGroup4.name()); - Assertions.assertNull(actualOverwriteGroup4.roleNames()); + Assertions.assertTrue(actualOverwriteGroup4.roles().isEmpty()); } @Test @@ -341,8 +336,7 @@ void deleteGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group2, false); List rolePOs = SessionUtils.doWithCommitAndFetchResult( @@ -350,8 +344,7 @@ void deleteGroup() throws IOException { Assertions.assertEquals(2, rolePOs.size()); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); Assertions.assertEquals(group2.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group2.roles()), Sets.newHashSet(actualGroup.roles())); Assertions.assertTrue(groupMetaService.deleteGroup(group2.nameIdentifier())); Assertions.assertThrows( @@ -400,13 +393,11 @@ void updateGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group1, false); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group1.nameIdentifier()); Assertions.assertEquals(group1.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group1.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group1.roles()), Sets.newHashSet(actualGroup.roles())); RoleEntity role3 = createRoleEntity( @@ -428,17 +419,14 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); - roleNames.add(role3.name()); - roleIds.add(role3.id()); + List roleEntities = Lists.newArrayList(group.roleEntities()); + roleEntities.add(role3); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -449,9 +437,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), grantGroup.id()); Assertions.assertEquals(group1.name(), grantGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantGroup.roleNames())); + Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role2.id(), role3.id()), Sets.newHashSet(grantGroup.roleIds())); + Sets.newHashSet(role1.id(), role2.id(), role3.id()), + grantGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantGroup.auditInfo().creator()); Assertions.assertEquals("grantGroup", grantGroup.auditInfo().lastModifier()); @@ -466,17 +455,16 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); - roleIds.remove(roleNames.indexOf("role2")); - roleNames.remove("role2"); + List roleEntities = Lists.newArrayList(group.roleEntities()); + List roleNames = + roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList()); + roleEntities.remove(roleNames.indexOf("role2")); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -487,9 +475,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), revokeGroup.id()); Assertions.assertEquals(group1.name(), revokeGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeGroup.roleNames())); + Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role3.id()), Sets.newHashSet(revokeGroup.roleIds())); + Sets.newHashSet(role1.id(), role3.id()), + revokeGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", revokeGroup.auditInfo().creator()); Assertions.assertEquals("revokeGroup", revokeGroup.auditInfo().lastModifier()); @@ -513,19 +502,16 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); - roleIds.remove(roleNames.indexOf("role3")); - roleNames.remove("role3"); - roleIds.add(role4.id()); - roleNames.add(role4.name()); + List roleNames = Lists.newArrayList(group.roles()); + List roleEntities = Lists.newArrayList(group.roleEntities()); + roleEntities.remove(roleNames.indexOf("role3")); + roleEntities.add(role4); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -536,9 +522,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), grantRevokeGroup.id()); Assertions.assertEquals(group1.name(), grantRevokeGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeGroup.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(grantRevokeGroup.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + grantRevokeGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantRevokeGroup.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", grantRevokeGroup.auditInfo().lastModifier()); @@ -553,15 +540,13 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); + List roleEntities = Lists.newArrayList(group.roleEntities()); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -571,9 +556,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), noUpdaterGroup.id()); Assertions.assertEquals(group1.name(), noUpdaterGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterGroup.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterGroup.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + noUpdaterGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", noUpdaterGroup.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", noUpdaterGroup.auditInfo().lastModifier()); @@ -581,7 +567,7 @@ void updateGroup() throws IOException { RoleMetaService.getInstance().deleteRole(role1.nameIdentifier()); GroupEntity groupEntity = GroupMetaService.getInstance().getGroupByIdentifier(group1.nameIdentifier()); - Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(groupEntity.roleNames())); + Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(groupEntity.roles())); } @Test @@ -629,16 +615,14 @@ void deleteMetalake() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); @@ -710,16 +694,14 @@ void deleteMetalakeCascade() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); @@ -782,32 +764,28 @@ void deleteGroupMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group3 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "group3", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group4 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "group4", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); groupMetaService.insertGroup(group3, false); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java index d811b8b598f..1db24de1c9b 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java @@ -301,41 +301,36 @@ void deleteRole() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role2.name()), - Lists.newArrayList(role2.id())); + Lists.newArrayList(role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role2.name()), - Lists.newArrayList(role2.id())); + Lists.newArrayList(role2)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); Assertions.assertEquals( group1.name(), groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).name()); Assertions.assertEquals( - group1.roleNames(), - groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).roleNames()); + group1.roles(), groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).roles()); Assertions.assertEquals( group2.name(), groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).name()); Assertions.assertEquals( - group2.roleNames(), - groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).roleNames()); + group2.roles(), groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).roles()); UserEntity user1 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role2.name()), - Lists.newArrayList(role2.id())); + Lists.newArrayList(role2)); userMetaService.insertUser(user1, false); Assertions.assertEquals( user1.name(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).name()); Assertions.assertEquals( - user1.roleNames(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).roleNames()); + user1.roles(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).roles()); Assertions.assertTrue(roleMetaService.deleteRole(role2.nameIdentifier())); Assertions.assertThrows( @@ -434,16 +429,14 @@ void deleteMetalake() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user1 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); groupMetaService.insertGroup(group1, false); @@ -546,16 +539,14 @@ void deleteMetalakeCascade() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user1 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); groupMetaService.insertGroup(group1, false); @@ -658,8 +649,7 @@ void deleteRoleMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user1, false); GroupEntity group1 = @@ -668,8 +658,7 @@ void deleteRoleMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group1, false); // hard delete before soft delete diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java index 326ccfc2da3..90e675e06df 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.Namespace; @@ -109,13 +110,11 @@ void getUserByIdentifier() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user2, false); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user2.roles()), Sets.newHashSet(actualUser.roles())); } @Test @@ -193,13 +192,11 @@ void insertUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user2, false)); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user2.roles()), Sets.newHashSet(actualUser.roles())); // insert duplicate user with roles UserEntity user2Exist = @@ -218,17 +215,15 @@ void insertUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2Overwrite", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user2Overwrite, true)); UserEntity actualOverwriteUser2 = userMetaService.getUserByIdentifier(user2Overwrite.nameIdentifier()); Assertions.assertEquals("user2Overwrite", actualOverwriteUser2.name()); - Assertions.assertEquals(2, actualOverwriteUser2.roleNames().size()); + Assertions.assertEquals(2, actualOverwriteUser2.roles().size()); Assertions.assertEquals( - Sets.newHashSet(role1.name(), role2.name()), - Sets.newHashSet(actualOverwriteUser2.roleNames())); + Sets.newHashSet(role1.name(), role2.name()), Sets.newHashSet(actualOverwriteUser2.roles())); // insert overwrite user with 1 role RoleEntity role3 = @@ -245,15 +240,14 @@ void insertUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user3Overwrite", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user3Overwrite, true)); UserEntity actualOverwriteUser3 = userMetaService.getUserByIdentifier(user3Overwrite.nameIdentifier()); Assertions.assertEquals("user3Overwrite", actualOverwriteUser3.name()); - Assertions.assertEquals(1, actualOverwriteUser3.roleNames().size()); - Assertions.assertEquals("role3", actualOverwriteUser3.roleNames().get(0)); + Assertions.assertEquals(1, actualOverwriteUser3.roles().size()); + Assertions.assertEquals("role3", actualOverwriteUser3.roles().get(0)); // insert overwrite user with 0 roles UserEntity user4Overwrite = @@ -267,7 +261,7 @@ void insertUser() throws IOException { UserEntity actualOverwriteUser4 = userMetaService.getUserByIdentifier(user4Overwrite.nameIdentifier()); Assertions.assertEquals("user4Overwrite", actualOverwriteUser4.name()); - Assertions.assertNull(actualOverwriteUser4.roleNames()); + Assertions.assertTrue(actualOverwriteUser4.roles().isEmpty()); } @Test @@ -338,8 +332,7 @@ void deleteUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user2, false); List rolePOs = SessionUtils.doWithCommitAndFetchResult( @@ -347,8 +340,7 @@ void deleteUser() throws IOException { Assertions.assertEquals(2, rolePOs.size()); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user2.roles()), Sets.newHashSet(actualUser.roles())); Assertions.assertTrue(userMetaService.deleteUser(user2.nameIdentifier())); Assertions.assertThrows( @@ -397,13 +389,11 @@ void updateUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user1, false); UserEntity actualUser = userMetaService.getUserByIdentifier(user1.nameIdentifier()); Assertions.assertEquals(user1.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user1.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user1.roles()), Sets.newHashSet(actualUser.roles())); RoleEntity role3 = createRoleEntity( @@ -425,17 +415,14 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); - roleNames.add(role3.name()); - roleIds.add(role3.id()); + List roleEntities = Lists.newArrayList(user.roleEntities()); + roleEntities.add(role3); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -446,9 +433,10 @@ void updateUser() throws IOException { Assertions.assertEquals(user1.id(), grantUser.id()); Assertions.assertEquals(user1.name(), grantUser.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantUser.roleNames())); + Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantUser.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role2.id(), role3.id()), Sets.newHashSet(grantUser.roleIds())); + Sets.newHashSet(role1.id(), role2.id(), role3.id()), + grantUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantUser.auditInfo().creator()); Assertions.assertEquals("grantUser", grantUser.auditInfo().lastModifier()); @@ -463,17 +451,15 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); - roleIds.remove(roleNames.indexOf("role2")); - roleNames.remove("role2"); + List roleNames = Lists.newArrayList(user.roles()); + List roleEntities = Lists.newArrayList(user.roleEntities()); + roleEntities.remove(roleNames.indexOf("role2")); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -483,10 +469,10 @@ void updateUser() throws IOException { UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); Assertions.assertEquals(user1.id(), revokeUser.id()); Assertions.assertEquals(user1.name(), revokeUser.name()); + Assertions.assertEquals(Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeUser.roles())); Assertions.assertEquals( - Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeUser.roleNames())); - Assertions.assertEquals( - Sets.newHashSet(role1.id(), role3.id()), Sets.newHashSet(revokeUser.roleIds())); + Sets.newHashSet(role1.id(), role3.id()), + revokeUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", revokeUser.auditInfo().creator()); Assertions.assertEquals("revokeUser", revokeUser.auditInfo().lastModifier()); @@ -510,19 +496,16 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); - roleIds.remove(roleNames.indexOf("role3")); - roleNames.remove("role3"); - roleIds.add(role4.id()); - roleNames.add(role4.name()); + List roleNames = Lists.newArrayList(user.roles()); + List roleEntities = Lists.newArrayList(user.roleEntities()); + roleEntities.remove(roleNames.indexOf("role3")); + roleEntities.add(role4); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -533,9 +516,10 @@ void updateUser() throws IOException { Assertions.assertEquals(user1.id(), grantRevokeUser.id()); Assertions.assertEquals(user1.name(), grantRevokeUser.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeUser.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeUser.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(grantRevokeUser.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + grantRevokeUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantRevokeUser.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", grantRevokeUser.auditInfo().lastModifier()); @@ -549,15 +533,13 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); + List roleEntities = Lists.newArrayList(user.roleEntities()); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -567,9 +549,10 @@ void updateUser() throws IOException { Assertions.assertEquals(user1.id(), noUpdaterUser.id()); Assertions.assertEquals(user1.name(), noUpdaterUser.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterUser.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterUser.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterUser.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + noUpdaterUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", noUpdaterUser.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", noUpdaterUser.auditInfo().lastModifier()); @@ -577,7 +560,7 @@ void updateUser() throws IOException { RoleMetaService.getInstance().deleteRole(role1.nameIdentifier()); UserEntity userEntity = UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); - Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(userEntity.roleNames())); + Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(userEntity.roles())); } @Test @@ -625,16 +608,14 @@ void deleteMetalake() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); @@ -706,16 +687,14 @@ void deleteMetalakeCascade() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); @@ -778,32 +757,28 @@ void deleteUserMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user3 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user3", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user4 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user4", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); userMetaService.insertUser(user3, false); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java index c3b34bc6bff..6a4b8e95354 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java @@ -245,7 +245,7 @@ private Group buildGroup(String group) { return GroupEntity.builder() .withId(1L) .withName(group) - .withRoleNames(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java index 76dba3edd4b..39ed06ebbcf 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java @@ -52,6 +52,7 @@ import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; +import org.apache.gravitino.meta.RoleEntity; import org.apache.gravitino.meta.UserEntity; import org.apache.gravitino.rest.RESTUtils; import org.glassfish.hk2.utilities.binding.AbstractBinder; @@ -110,12 +111,18 @@ protected void configure() { @Test public void testGrantRolesToUser() { + RoleEntity roleEntity = + RoleEntity.builder() + .withId(1L) + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .withName("roles") + .build(); UserEntity userEntity = UserEntity.builder() .withId(1L) .withName("user") - .withRoleNames(Lists.newArrayList("roles")) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList(roleEntity)) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); @@ -207,12 +214,18 @@ public void testGrantRolesToUser() { @Test public void testGrantRolesToGroup() { + RoleEntity roleEntity = + RoleEntity.builder() + .withId(1L) + .withName("roles") + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); GroupEntity groupEntity = GroupEntity.builder() .withId(1L) .withName("group") - .withRoleNames(Lists.newArrayList("roles")) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList(roleEntity)) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); @@ -311,8 +324,7 @@ public void testRevokeRolesFromUser() { UserEntity.builder() .withId(1L) .withName("user") - .withRoleNames(Lists.newArrayList()) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); @@ -356,8 +368,7 @@ public void testRevokeRolesFromGroup() { GroupEntity.builder() .withId(1L) .withName("group") - .withRoleNames(Lists.newArrayList()) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java index d3209e0e22c..b13c0175b0b 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java @@ -246,7 +246,7 @@ private User buildUser(String user) { return UserEntity.builder() .withId(1L) .withName(user) - .withRoleNames(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); From c7200c89a04ed8dab6f3bfe4d2f209220dd5415d Mon Sep 17 00:00:00 2001 From: mchades Date: Sun, 8 Sep 2024 06:25:52 +0800 Subject: [PATCH 6/8] [HOTFIX] fix(openapi): fix the env of openAPI lint plugin (#4876) ### What changes were proposed in this pull request? - update redocly/cli version from 1.5.0 to 1.23.1 - fixed the node version to 21.6.1 - fixed the npm version to 10.2.4 ### Why are the changes needed? redocly/cli failed at version 1.5.0 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? build CI passed --- .github/workflows/build.yml | 1 + docs/build.gradle.kts | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cf17683a0d..b934c4cc26a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,7 @@ jobs: - trino-connector/** - web/** - docs/open-api/** + - docs/build.gradle.kts - build.gradle.kts - gradle.properties - gradlew diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 16b5f2525fa..60b0be907a5 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -17,11 +17,18 @@ * under the License. */ +import com.github.gradle.node.NodeExtension import com.github.gradle.node.npm.task.NpxTask +configure { + version = "21.6.1" + npmVersion = "10.2.4" + download = true +} + tasks { val lintOpenAPI by registering(NpxTask::class) { - command.set("@redocly/cli@1.5.0") + command.set("@redocly/cli@1.23.1") args.set(listOf("lint", "--extends=recommended-strict", "${project.projectDir}/open-api/openapi.yaml")) } From 2a5e7d4009ab5164bd03a1d591e3e1e05321c1be Mon Sep 17 00:00:00 2001 From: Yuhui Date: Sun, 8 Sep 2024 10:36:02 +0800 Subject: [PATCH 7/8] (#4871)fix(test): Fix the jvm args `-Xmx4G` of gradle test can be overwrite by the `extraJvmArgs`. (#4872) ### What changes were proposed in this pull request? Fix the oom error when run the gradle test. ### Why are the changes needed? Fix: #4871 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Manually Co-authored-by: Jerry Shao --- build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 3e47891f84f..cb7bad9a851 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -431,9 +431,9 @@ subprojects { reports.html.outputLocation.set(file("${rootProject.projectDir}/build/reports/")) val skipTests = project.hasProperty("skipTests") if (!skipTests) { - jvmArgs = listOf("-Xmx4G") + val extraArgs = project.property("extraJvmArgs") as List + jvmArgs = listOf("-Xmx4G") + extraArgs useJUnitPlatform() - jvmArgs(project.property("extraJvmArgs") as List<*>) finalizedBy(tasks.getByName("jacocoTestReport")) } } From 60f4bebbc6eec4ef16ed10ad7530af09d7c34985 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Sun, 8 Sep 2024 11:38:24 +0800 Subject: [PATCH 8/8] [#4101] feat(core): Support PostgreSQL storage backend (#4611) ### What changes were proposed in this pull request? - Support PostgreSQL JDBC backend. ### Why are the changes needed? User need. Fix: #4101 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Existing ITs --------- Co-authored-by: Jerry Shao --- .../workflows/backend-integration-test.yml | 18 +- catalogs/catalog-hadoop/build.gradle.kts | 1 + catalogs/catalog-hive/build.gradle.kts | 1 + catalogs/catalog-jdbc-doris/build.gradle.kts | 1 + catalogs/catalog-jdbc-mysql/build.gradle.kts | 1 + catalogs/catalog-kafka/build.gradle.kts | 1 + .../catalog-lakehouse-paimon/build.gradle.kts | 1 + .../PostgreSQLExceptionConverter.java | 47 ++ .../SQLExceptionConverterFactory.java | 2 + .../mapper/CatalogMetaSQLProviderFactory.java | 4 +- .../mapper/FilesetMetaSQLProviderFactory.java | 4 +- .../FilesetVersionSQLProviderFactory.java | 4 +- .../mapper/GroupMetaSQLProviderFactory.java | 4 +- .../GroupRoleRelSQLProviderFactory.java | 4 +- .../MetalakeMetaSQLProviderFactory.java | 4 +- .../mapper/OwnerMetaSQLProviderFactory.java | 4 +- .../mapper/RoleMetaSQLProviderFactory.java | 4 +- .../mapper/SchemaMetaSQLProviderFactory.java | 4 +- .../SecurableObjectSQLProviderFactory.java | 4 +- .../mapper/TableMetaSQLProviderFactory.java | 4 +- .../mapper/TagMetaSQLProviderFactory.java | 4 +- ...agMetadataObjectRelSQLProviderFactory.java | 4 +- .../mapper/TopicMetaSQLProviderFactory.java | 4 +- .../mapper/UserMetaSQLProviderFactory.java | 4 +- .../mapper/UserRoleRelSQLProviderFactory.java | 4 +- .../CatalogMetaPostgreSQLProvider.java | 108 +++++ .../FilesetMetaPostgreSQLProvider.java | 93 ++++ .../FilesetVersionPostgreSQLProvider.java | 102 ++++ .../GroupMetaPostgreSQLProvider.java | 69 +++ .../GroupRoleRelPostgreSQLProvider.java | 73 +++ .../MetalakeMetaPostgreSQLProvider.java | 89 ++++ .../OwnerMetaPostgreSQLProvider.java | 120 +++++ .../RoleMetaPostgreSQLProvider.java | 70 +++ .../SchemaMetaPostgreSQLProvider.java | 84 ++++ .../SecurableObjectPostgreSQLProvider.java | 47 ++ .../TableMetaPostgreSQLProvider.java | 91 ++++ .../postgresql/TagMetaPostgreSQLProvider.java | 102 ++++ ...agMetadataObjectRelPostgreSQLProvider.java | 89 ++++ .../TopicMetaPostgreSQLProvider.java | 96 ++++ .../UserMetaPostgreSQLProvider.java | 69 +++ .../UserRoleRelPostgreSQLProvider.java | 102 ++++ .../test/container/PostgreSQLContainer.java | 2 +- .../integration/test/util/AbstractIT.java | 70 +++ .../test/util/TestDatabaseName.java | 2 + .../postgresql/schema-0.7.0-postgresql.sql | 458 ++++++++++++++++++ 45 files changed, 2050 insertions(+), 23 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/converters/PostgreSQLExceptionConverter.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/CatalogMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetVersionPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupRoleRelPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/MetalakeMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/OwnerMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/RoleMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SchemaMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SecurableObjectPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TableMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetadataObjectRelPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TopicMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserMetaPostgreSQLProvider.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserRoleRelPostgreSQLProvider.java create mode 100644 scripts/postgresql/schema-0.7.0-postgresql.sql diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 1c6596f772a..c4c0a446759 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -61,12 +61,15 @@ jobs: # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 8, 11, 17 ] + backend: [ h2, mysql, postgresql ] test-mode: [ embedded, deploy ] - include: + exclude: - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' backend: 'mysql' + - test-mode: 'embedded' + backend: 'postgresql' + - test-mode: 'deploy' + backend: 'h2' env: PLATFORM: ${{ matrix.architecture }} @@ -132,11 +135,14 @@ jobs: architecture: [ linux/amd64 ] java-version: [ 17 ] test-mode: [ embedded, deploy ] - include: + backend: [ h2, mysql, postgresql ] + exclude: - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' backend: 'mysql' + - test-mode: 'embedded' + backend: 'postgresql' + - test-mode: 'deploy' + backend: 'h2' env: PLATFORM: ${{ matrix.architecture }} diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index ea38a895f08..429cd405243 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -79,6 +79,7 @@ dependencies { testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 776e9bf39c9..081bfcbeb6e 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -132,6 +132,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") { exclude("org.apache.hadoop") diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index b7cf6dfc871..ccfaaddfb7b 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -52,6 +52,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) diff --git a/catalogs/catalog-jdbc-mysql/build.gradle.kts b/catalogs/catalog-jdbc-mysql/build.gradle.kts index 9ad6c739ab3..95f0578d3bf 100644 --- a/catalogs/catalog-jdbc-mysql/build.gradle.kts +++ b/catalogs/catalog-jdbc-mysql/build.gradle.kts @@ -55,6 +55,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts index a2493fa77d9..3165758a168 100644 --- a/catalogs/catalog-kafka/build.gradle.kts +++ b/catalogs/catalog-kafka/build.gradle.kts @@ -51,6 +51,7 @@ dependencies { testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts index a6adf999d31..930e3e3354d 100644 --- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts +++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts @@ -78,6 +78,7 @@ dependencies { testImplementation(libs.slf4j.api) testImplementation(libs.junit.jupiter.api) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.bundles.log4j) testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/converters/PostgreSQLExceptionConverter.java b/core/src/main/java/org/apache/gravitino/storage/relational/converters/PostgreSQLExceptionConverter.java new file mode 100644 index 00000000000..7414760930d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/converters/PostgreSQLExceptionConverter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.converters; + +import java.io.IOException; +import java.sql.SQLException; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; + +/** + * Exception converter to Apache Gravitino exception for PostgreSQL. The definition of error codes + * can be found in the document: error code of PostgreSQL + */ +public class PostgreSQLExceptionConverter implements SQLExceptionConverter { + private static final int DUPLICATED_ENTRY_ERROR_CODE = 23505; + + @Override + @SuppressWarnings("FormatStringAnnotation") + public void toGravitinoException(SQLException sqlException, Entity.EntityType type, String name) + throws IOException { + int errorCode = Integer.valueOf(sqlException.getSQLState()); + switch (errorCode) { + case DUPLICATED_ENTRY_ERROR_CODE: + throw new EntityAlreadyExistsException( + sqlException, "The %s entity: %s already exists.", type.name(), name); + default: + throw new IOException(sqlException); + } + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java index 4954910c9ec..feb0fb0d7df 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java @@ -40,6 +40,8 @@ public static synchronized void initConverter(Config config) { converter = new MySQLExceptionConverter(); } else if (jdbcType.equalsIgnoreCase("h2")) { converter = new H2ExceptionConverter(); + } else if (jdbcType.equalsIgnoreCase("postgresql")) { + converter = new PostgreSQLExceptionConverter(); } else { throw new IllegalArgumentException(String.format("Unsupported jdbc type: %s", jdbcType)); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java index 5c0e63f531b..632681c5f0c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.CatalogMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.CatalogPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -32,7 +33,8 @@ public class CatalogMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new CatalogMetaMySQLProvider(), - JDBCBackendType.H2, new CatalogMetaH2Provider()); + JDBCBackendType.H2, new CatalogMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new CatalogMetaPostgreSQLProvider()); public static CatalogMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java index 36ea94d5862..b41237e2a79 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.FilesetMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.FilesetPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -31,7 +32,8 @@ public class FilesetMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new FilesetMetaMySQLProvider(), - JDBCBackendType.H2, new FilesetMetaH2Provider()); + JDBCBackendType.H2, new FilesetMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new FilesetMetaPostgreSQLProvider()); public static FilesetMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java index 163f2c882fe..4029c2cb055 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.FilesetVersionPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.FilesetVersionPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -30,7 +31,8 @@ public class FilesetVersionSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new FilesetVersionMySQLProvider(), - JDBCBackendType.H2, new FilesetVersionH2Provider()); + JDBCBackendType.H2, new FilesetVersionH2Provider(), + JDBCBackendType.POSTGRESQL, new FilesetVersionPostgreSQLProvider()); public static FilesetVersionBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java index 59e45a6dc6b..2769bf1d959 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.GroupMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.GroupPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -30,7 +31,8 @@ public class GroupMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new GroupMetaMySQLProvider(), - JDBCBackendType.H2, new GroupMetaH2Provider()); + JDBCBackendType.H2, new GroupMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new GroupMetaPostgreSQLProvider()); public static GroupMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java index 4fd047abeb1..6d2ff176bd1 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.GroupRoleRelPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -32,7 +33,8 @@ public class GroupRoleRelSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new GroupRoleRelMySQLProvider(), - JDBCBackendType.H2, new GroupRoleRelH2Provider()); + JDBCBackendType.H2, new GroupRoleRelH2Provider(), + JDBCBackendType.POSTGRESQL, new GroupRoleRelPostgreSQLProvider()); public static GroupRoleRelBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java index e28cbc9d774..67589459631 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.MetalakeMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.MetalakePO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -32,7 +33,8 @@ public class MetalakeMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new MetalakeMetaMySQLProvider(), - JDBCBackendType.H2, new MetalakeMetaH2Provider()); + JDBCBackendType.H2, new MetalakeMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new MetalakeMetaPostgreSQLProvider()); public static MetalakeMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java index e07fd269b77..8ddf53d5d50 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.OwnerMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.OwnerRelPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -31,7 +32,8 @@ public class OwnerMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new OwnerMetaMySQLProvider(), - JDBCBackendType.H2, new OwnerMetaH2Provider()); + JDBCBackendType.H2, new OwnerMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new OwnerMetaPostgreSQLProvider()); public static OwnerMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java index bdcb45749da..41599386003 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.RoleMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -30,7 +31,8 @@ public class RoleMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new RoleMetaMySQLProvider(), - JDBCBackendType.H2, new RoleMetaH2Provider()); + JDBCBackendType.H2, new RoleMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new RoleMetaPostgreSQLProvider()); public static RoleMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java index 5fa6252d5b6..87f636e61d8 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.SchemaMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.SchemaPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -30,7 +31,8 @@ public class SchemaMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new SchemaMetaMySQLProvider(), - JDBCBackendType.H2, new SchemaMetaH2Provider()); + JDBCBackendType.H2, new SchemaMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new SchemaMetaPostgreSQLProvider()); public static SchemaMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java index 6508d7db39c..da4ddc730a0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.SecurableObjectPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.SecurableObjectPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -32,7 +33,8 @@ public class SecurableObjectSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new SecurableObjectMySQLProvider(), - JDBCBackendType.H2, new SecurableObjectH2Provider()); + JDBCBackendType.H2, new SecurableObjectH2Provider(), + JDBCBackendType.POSTGRESQL, new SecurableObjectPostgreSQLProvider()); public static SecurableObjectBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java index 833ba9a059d..619648c1f52 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TableMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.TablePO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -31,7 +32,8 @@ public class TableMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new TableMetaMySQLProvider(), - JDBCBackendType.H2, new TableMetaH2Provider()); + JDBCBackendType.H2, new TableMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new TableMetaPostgreSQLProvider()); public static TableMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java index aaa92b038ef..c9a018c5f2c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TagMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.TagPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -31,7 +32,8 @@ public class TagMetaSQLProviderFactory { private static final Map METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new TagMetaMySQLProvider(), - JDBCBackendType.H2, new TagMetaH2Provider()); + JDBCBackendType.H2, new TagMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new TagMetaPostgreSQLProvider()); public static TagMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java index b074349e817..af4522f9d95 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TagMetadataObjectRelPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -32,7 +33,8 @@ public class TagMetadataObjectRelSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new TagMetadataObjectRelMySQLProvider(), - JDBCBackendType.H2, new TagMetadataObjectRelH2Provider()); + JDBCBackendType.H2, new TagMetadataObjectRelH2Provider(), + JDBCBackendType.POSTGRESQL, new TagMetadataObjectRelPostgreSQLProvider()); public static TagMetadataObjectRelBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java index 9a417e011a7..c1c6e2e237d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TopicMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.TopicPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -32,7 +33,8 @@ public class TopicMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new TopicMetaMySQLProvider(), - JDBCBackendType.H2, new TopicMetaH2Provider()); + JDBCBackendType.H2, new TopicMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new TopicMetaPostgreSQLProvider()); public static TopicMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java index 3c64f510c27..2c322db86d6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.UserMetaPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.UserPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -32,7 +33,8 @@ public class UserMetaSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new UserMetaMySQLProvider(), - JDBCBackendType.H2, new UserMetaH2Provider()); + JDBCBackendType.H2, new UserMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new UserMetaPostgreSQLProvider()); public static UserMetaBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java index 3d52c6aa02c..f98f509f8f2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.UserRoleRelPostgreSQLProvider; import org.apache.gravitino.storage.relational.po.UserRoleRelPO; import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; import org.apache.ibatis.annotations.Param; @@ -33,7 +34,8 @@ public class UserRoleRelSQLProviderFactory { METALAKE_META_SQL_PROVIDER_MAP = ImmutableMap.of( JDBCBackendType.MYSQL, new UserRoleRelMySQLProvider(), - JDBCBackendType.H2, new UserRoleRelH2Provider()); + JDBCBackendType.H2, new UserRoleRelH2Provider(), + JDBCBackendType.POSTGRESQL, new UserRoleRelPostgreSQLProvider()); public static UserRoleRelBaseSQLProvider getProvider() { String databaseId = diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/CatalogMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/CatalogMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..a4646fde852 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/CatalogMetaPostgreSQLProvider.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.CatalogMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.CatalogPO; +import org.apache.ibatis.annotations.Param; + +public class CatalogMetaPostgreSQLProvider extends CatalogMetaBaseSQLProvider { + @Override + public String softDeleteCatalogMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteCatalogMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertCatalogMetaOnDuplicateKeyUpdate(CatalogPO catalogPO) { + return "INSERT INTO " + + TABLE_NAME + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{catalogMeta.catalogId}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + + " )" + + " ON CONFLICT(catalog_id) DO UPDATE SET" + + " catalog_name = #{catalogMeta.catalogName}," + + " metalake_id = #{catalogMeta.metalakeId}," + + " type = #{catalogMeta.type}," + + " provider = #{catalogMeta.provider}," + + " catalog_comment = #{catalogMeta.catalogComment}," + + " properties = #{catalogMeta.properties}," + + " audit_info = #{catalogMeta.auditInfo}," + + " current_version = #{catalogMeta.currentVersion}," + + " last_version = #{catalogMeta.lastVersion}," + + " deleted_at = #{catalogMeta.deletedAt}"; + } + + public String updateCatalogMeta( + @Param("newCatalogMeta") CatalogPO newCatalogPO, + @Param("oldCatalogMeta") CatalogPO oldCatalogPO) { + return "UPDATE " + + TABLE_NAME + + " SET catalog_name = #{newCatalogMeta.catalogName}," + + " metalake_id = #{newCatalogMeta.metalakeId}," + + " type = #{newCatalogMeta.type}," + + " provider = #{newCatalogMeta.provider}," + + " catalog_comment = #{newCatalogMeta.catalogComment}," + + " properties = #{newCatalogMeta.properties}," + + " audit_info = #{newCatalogMeta.auditInfo}," + + " current_version = #{newCatalogMeta.currentVersion}," + + " last_version = #{newCatalogMeta.lastVersion}," + + " deleted_at = #{newCatalogMeta.deletedAt}" + + " WHERE catalog_id = #{oldCatalogMeta.catalogId}" + + " AND catalog_name = #{oldCatalogMeta.catalogName}" + + " AND metalake_id = #{oldCatalogMeta.metalakeId}" + + " AND type = #{oldCatalogMeta.type}" + + " AND provider = #{oldCatalogMeta.provider}" + + " AND (catalog_comment = #{oldCatalogMeta.catalogComment} " + + " OR (CAST(catalog_comment AS VARCHAR) IS NULL AND " + + " CAST(#{oldCatalogMeta.catalogComment} AS VARCHAR) IS NULL))" + + " AND properties = #{oldCatalogMeta.properties}" + + " AND audit_info = #{oldCatalogMeta.auditInfo}" + + " AND current_version = #{oldCatalogMeta.currentVersion}" + + " AND last_version = #{oldCatalogMeta.lastVersion}" + + " AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..b63314088cd --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetMetaPostgreSQLProvider.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper.META_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.FilesetMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.FilesetPO; + +public class FilesetMetaPostgreSQLProvider extends FilesetMetaBaseSQLProvider { + @Override + public String softDeleteFilesetMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetMetasByCatalogId(Long catalogId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetMetasBySchemaId(Long schemaId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetMetasByFilesetId(Long filesetId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + @Override + public String insertFilesetMetaOnDuplicateKeyUpdate(FilesetPO filesetPO) { + return "INSERT INTO " + + META_TABLE_NAME + + "(fileset_id, fileset_name, metalake_id," + + " catalog_id, schema_id, type, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{filesetMeta.filesetId}," + + " #{filesetMeta.filesetName}," + + " #{filesetMeta.metalakeId}," + + " #{filesetMeta.catalogId}," + + " #{filesetMeta.schemaId}," + + " #{filesetMeta.type}," + + " #{filesetMeta.auditInfo}," + + " #{filesetMeta.currentVersion}," + + " #{filesetMeta.lastVersion}," + + " #{filesetMeta.deletedAt}" + + " )" + + " ON CONFLICT(fileset_id) DO UPDATE SET" + + " fileset_name = #{filesetMeta.filesetName}," + + " metalake_id = #{filesetMeta.metalakeId}," + + " catalog_id = #{filesetMeta.catalogId}," + + " schema_id = #{filesetMeta.schemaId}," + + " type = #{filesetMeta.type}," + + " audit_info = #{filesetMeta.auditInfo}," + + " current_version = #{filesetMeta.currentVersion}," + + " last_version = #{filesetMeta.lastVersion}," + + " deleted_at = #{filesetMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetVersionPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetVersionPostgreSQLProvider.java new file mode 100644 index 00000000000..55317c0c755 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetVersionPostgreSQLProvider.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper.VERSION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.FilesetVersionBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.FilesetVersionPO; + +public class FilesetVersionPostgreSQLProvider extends FilesetVersionBaseSQLProvider { + @Override + public String softDeleteFilesetVersionsByMetalakeId(Long metalakeId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsByCatalogId(Long catalogId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsBySchemaId(Long schemaId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsByFilesetId(Long filesetId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsByRetentionLine( + Long filesetId, long versionRetentionLine, int limit) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}"; + } + + @Override + public String insertFilesetVersionOnDuplicateKeyUpdate(FilesetVersionPO filesetVersionPO) { + return "INSERT INTO " + + VERSION_TABLE_NAME + + "(metalake_id, catalog_id, schema_id, fileset_id," + + " version, fileset_comment, properties, storage_location," + + " deleted_at)" + + " VALUES(" + + " #{filesetVersion.metalakeId}," + + " #{filesetVersion.catalogId}," + + " #{filesetVersion.schemaId}," + + " #{filesetVersion.filesetId}," + + " #{filesetVersion.version}," + + " #{filesetVersion.filesetComment}," + + " #{filesetVersion.properties}," + + " #{filesetVersion.storageLocation}," + + " #{filesetVersion.deletedAt}" + + " )" + + " ON CONFLICT(fileset_id, version, deleted_at) DO UPDATE SET" + + " metalake_id = #{filesetVersion.metalakeId}," + + " catalog_id = #{filesetVersion.catalogId}," + + " schema_id = #{filesetVersion.schemaId}," + + " fileset_id = #{filesetVersion.filesetId}," + + " version = #{filesetVersion.version}," + + " fileset_comment = #{filesetVersion.filesetComment}," + + " properties = #{filesetVersion.properties}," + + " storage_location = #{filesetVersion.storageLocation}," + + " deleted_at = #{filesetVersion.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..0e24b319ed8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupMetaPostgreSQLProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.GroupMetaMapper.GROUP_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.GroupMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.GroupPO; + +public class GroupMetaPostgreSQLProvider extends GroupMetaBaseSQLProvider { + @Override + public String softDeleteGroupMetaByGroupId(Long groupId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + @Override + public String softDeleteGroupMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertGroupMetaOnDuplicateKeyUpdate(GroupPO groupPO) { + return "INSERT INTO " + + GROUP_TABLE_NAME + + "(group_id, group_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )" + + " ON CONFLICT(group_id) DO UPDATE SET" + + " group_name = #{groupMeta.groupName}," + + " metalake_id = #{groupMeta.metalakeId}," + + " audit_info = #{groupMeta.auditInfo}," + + " current_version = #{groupMeta.currentVersion}," + + " last_version = #{groupMeta.lastVersion}," + + " deleted_at = #{groupMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupRoleRelPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupRoleRelPostgreSQLProvider.java new file mode 100644 index 00000000000..02f81bc73e8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupRoleRelPostgreSQLProvider.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.mapper.GroupRoleRelBaseSQLProvider; + +public class GroupRoleRelPostgreSQLProvider extends GroupRoleRelBaseSQLProvider { + @Override + public String softDeleteGroupRoleRelByGroupId(Long groupId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + @Override + public String softDeleteGroupRoleRelByGroupAndRoles(Long groupId, List roleIds) { + return ""; + } + + @Override + public String softDeleteGroupRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE group_id IN (SELECT group_id FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + @Override + public String softDeleteGroupRoleRelByRoleId(Long roleId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/MetalakeMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/MetalakeMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..5375f74c0da --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/MetalakeMetaPostgreSQLProvider.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.ibatis.annotations.Param; + +public class MetalakeMetaPostgreSQLProvider extends MetalakeMetaBaseSQLProvider { + @Override + public String softDeleteMetalakeMetaByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertMetalakeMetaOnDuplicateKeyUpdate(MetalakePO metalakePO) { + return "INSERT INTO " + + TABLE_NAME + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{metalakeMeta.metalakeId}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + + " )" + + " ON CONFLICT(metalake_id) DO UPDATE SET" + + " metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}," + + " current_version = #{metalakeMeta.currentVersion}," + + " last_version = #{metalakeMeta.lastVersion}," + + " deleted_at = #{metalakeMeta.deletedAt}"; + } + + public String updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO) { + return "UPDATE " + + TABLE_NAME + + " SET metalake_name = #{newMetalakeMeta.metalakeName}," + + " metalake_comment = #{newMetalakeMeta.metalakeComment}," + + " properties = #{newMetalakeMeta.properties}," + + " audit_info = #{newMetalakeMeta.auditInfo}," + + " schema_version = #{newMetalakeMeta.schemaVersion}," + + " current_version = #{newMetalakeMeta.currentVersion}," + + " last_version = #{newMetalakeMeta.lastVersion}" + + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" + + " AND metalake_name = #{oldMetalakeMeta.metalakeName}" + + " AND (metalake_comment = #{oldMetalakeMeta.metalakeComment} " + + " OR (CAST(metalake_comment AS VARCHAR) IS NULL AND " + + " CAST(#{oldMetalakeMeta.metalakeComment} AS VARCHAR) IS NULL))" + + " AND properties = #{oldMetalakeMeta.properties}" + + " AND audit_info = #{oldMetalakeMeta.auditInfo}" + + " AND schema_version = #{oldMetalakeMeta.schemaVersion}" + + " AND current_version = #{oldMetalakeMeta.currentVersion}" + + " AND last_version = #{oldMetalakeMeta.lastVersion}" + + " AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/OwnerMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/OwnerMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..6374a3fbd28 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/OwnerMetaPostgreSQLProvider.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper.OWNER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; +import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; +import org.apache.gravitino.storage.relational.mapper.OwnerMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; + +public class OwnerMetaPostgreSQLProvider extends OwnerMetaBaseSQLProvider { + @Override + public String softDeleteOwnerRelByMetadataObjectIdAndType( + Long metadataObjectId, String metadataObjectType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metadata_object_id = #{metadataObjectId} AND metadata_object_type = #{metadataObjectType} AND deleted_at = 0"; + } + + @Override + public String softDeleteOwnerRelByOwnerIdAndType(Long ownerId, String ownerType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE owner_id = #{ownerId} AND owner_type = #{ownerType} AND deleted_at = 0"; + } + + @Override + public String softDeleteOwnerRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at =0"; + } + + @Override + public String softDeleteOwnerRelByCatalogId(Long catalogId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE EXISTS (" + + " SELECT ct.catalog_id FROM " + + CatalogMetaMapper.TABLE_NAME + + " ct WHERE ct.catalog_id = #{catalogId} AND ct.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'" + + " UNION " + + " SELECT st.catalog_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.catalog_id = #{catalogId} AND st.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.catalog_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.catalog_id = #{catalogId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.catalog_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.catalog_id = #{catalogId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.catalog_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.catalog_id = #{catalogId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND" + + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } + + @Override + public String sotDeleteOwnerRelBySchemaId(Long schemaId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET deleted_at = floor(extract(epoch from((current_timestamp - timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE EXISTS (" + + " SELECT st.schema_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.schema_id = #{schemaId} AND st.deleted_at = 0 AND ot.deleted_at = 0 " + + "AND st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.schema_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.schema_id = #{schemaId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.schema_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.schema_id = #{schemaId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.schema_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.schema_id = #{schemaId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/RoleMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/RoleMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..72e5808e37d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/RoleMetaPostgreSQLProvider.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.ROLE_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.RoleMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.RolePO; + +public class RoleMetaPostgreSQLProvider extends RoleMetaBaseSQLProvider { + @Override + public String softDeleteRoleMetaByRoleId(Long roleId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + @Override + public String softDeleteRoleMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertRoleMetaOnDuplicateKeyUpdate(RolePO rolePO) { + return "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " ) ON CONFLICT (role_id) DO UPDATE SET" + + " role_name = #{roleMeta.roleName}," + + " metalake_id = #{roleMeta.metalakeId}," + + " properties = #{roleMeta.properties}," + + " audit_info = #{roleMeta.auditInfo}," + + " current_version = #{roleMeta.currentVersion}," + + " last_version = #{roleMeta.lastVersion}," + + " deleted_at = #{roleMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SchemaMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SchemaMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..c2a565ddd08 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SchemaMetaPostgreSQLProvider.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.SchemaMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.SchemaPO; + +public class SchemaMetaPostgreSQLProvider extends SchemaMetaBaseSQLProvider { + @Override + public String insertSchemaMetaOnDuplicateKeyUpdate(SchemaPO schemaPO) { + return "INSERT INTO " + + TABLE_NAME + + "(schema_id, schema_name, metalake_id," + + " catalog_id, schema_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{schemaMeta.schemaId}," + + " #{schemaMeta.schemaName}," + + " #{schemaMeta.metalakeId}," + + " #{schemaMeta.catalogId}," + + " #{schemaMeta.schemaComment}," + + " #{schemaMeta.properties}," + + " #{schemaMeta.auditInfo}," + + " #{schemaMeta.currentVersion}," + + " #{schemaMeta.lastVersion}," + + " #{schemaMeta.deletedAt}" + + " )" + + " ON CONFLICT(schema_id) DO UPDATE SET " + + " schema_name = #{schemaMeta.schemaName}," + + " metalake_id = #{schemaMeta.metalakeId}," + + " catalog_id = #{schemaMeta.catalogId}," + + " schema_comment = #{schemaMeta.schemaComment}," + + " properties = #{schemaMeta.properties}," + + " audit_info = #{schemaMeta.auditInfo}," + + " current_version = #{schemaMeta.currentVersion}," + + " last_version = #{schemaMeta.lastVersion}," + + " deleted_at = #{schemaMeta.deletedAt}"; + } + + @Override + public String softDeleteSchemaMetasBySchemaId(Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String softDeleteSchemaMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteSchemaMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SecurableObjectPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SecurableObjectPostgreSQLProvider.java new file mode 100644 index 00000000000..ff1340e6c2c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SecurableObjectPostgreSQLProvider.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.ROLE_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.SecurableObjectBaseSQLProvider; + +public class SecurableObjectPostgreSQLProvider extends SecurableObjectBaseSQLProvider { + @Override + public String softDeleteSecurableObjectsByRoleId(Long roleId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + @Override + public String softDeleteRoleMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " ob SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE exists (SELECT * from " + + ROLE_TABLE_NAME + + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" + + " AND ro.deleted_at = 0) AND ob.deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TableMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TableMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..d1f5d398591 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TableMetaPostgreSQLProvider.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.TableMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.TablePO; + +public class TableMetaPostgreSQLProvider extends TableMetaBaseSQLProvider { + @Override + public String insertTableMetaOnDuplicateKeyUpdate(TablePO tablePO) { + return "INSERT INTO " + + TABLE_NAME + + "(table_id, table_name, metalake_id," + + " catalog_id, schema_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tableMeta.tableId}," + + " #{tableMeta.tableName}," + + " #{tableMeta.metalakeId}," + + " #{tableMeta.catalogId}," + + " #{tableMeta.schemaId}," + + " #{tableMeta.auditInfo}," + + " #{tableMeta.currentVersion}," + + " #{tableMeta.lastVersion}," + + " #{tableMeta.deletedAt}" + + " )" + + " ON CONFLICT (table_id) DO UPDATE SET " + + " table_name = #{tableMeta.tableName}," + + " metalake_id = #{tableMeta.metalakeId}," + + " catalog_id = #{tableMeta.catalogId}," + + " schema_id = #{tableMeta.schemaId}," + + " audit_info = #{tableMeta.auditInfo}," + + " current_version = #{tableMeta.currentVersion}," + + " last_version = #{tableMeta.lastVersion}," + + " deleted_at = #{tableMeta.deletedAt}"; + } + + @Override + public String softDeleteTableMetasByTableId(Long tableId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE table_id = #{tableId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTableMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTableMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTableMetasBySchemaId(Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..75a3863f7ba --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetaPostgreSQLProvider.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TagMetaMapper.TAG_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.TagPO; +import org.apache.ibatis.annotations.Param; + +public class TagMetaPostgreSQLProvider extends TagMetaBaseSQLProvider { + @Override + public String softDeleteTagMetaByMetalakeAndTagName(String metalakeName, String tagName) { + return "UPDATE " + + TAG_TABLE_NAME + + " tm SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE tm.metalake_id IN (" + + " SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0"; + } + + @Override + public String softDeleteTagMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertTagMetaOnDuplicateKeyUpdate(TagPO tagPO) { + return "INSERT INTO " + + TAG_TABLE_NAME + + "(tag_id, tag_name," + + " metalake_id, tag_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tagMeta.tagId}," + + " #{tagMeta.tagName}," + + " #{tagMeta.metalakeId}," + + " #{tagMeta.comment}," + + " #{tagMeta.properties}," + + " #{tagMeta.auditInfo}," + + " #{tagMeta.currentVersion}," + + " #{tagMeta.lastVersion}," + + " #{tagMeta.deletedAt}" + + " )" + + " ON CONFLICT(tag_id) DO UPDATE SET" + + " tag_name = #{tagMeta.tagName}," + + " metalake_id = #{tagMeta.metalakeId}," + + " tag_comment = #{tagMeta.comment}," + + " properties = #{tagMeta.properties}," + + " audit_info = #{tagMeta.auditInfo}," + + " current_version = #{tagMeta.currentVersion}," + + " last_version = #{tagMeta.lastVersion}," + + " deleted_at = #{tagMeta.deletedAt}"; + } + + public String updateTagMeta( + @Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET tag_name = #{newTagMeta.tagName}," + + " tag_comment = #{newTagMeta.comment}," + + " properties = #{newTagMeta.properties}," + + " audit_info = #{newTagMeta.auditInfo}," + + " current_version = #{newTagMeta.currentVersion}," + + " last_version = #{newTagMeta.lastVersion}," + + " deleted_at = #{newTagMeta.deletedAt}" + + " WHERE tag_id = #{oldTagMeta.tagId}" + + " AND metalake_id = #{oldTagMeta.metalakeId}" + + " AND tag_name = #{oldTagMeta.tagName}" + + " AND (tag_comment = #{oldTagMeta.comment} " + + " OR (CAST(tag_comment AS VARCHAR) IS NULL AND CAST(#{oldTagMeta.comment} AS VARCHAR) IS NULL))" + + " AND properties = #{oldTagMeta.properties}" + + " AND audit_info = #{oldTagMeta.auditInfo}" + + " AND current_version = #{oldTagMeta.currentVersion}" + + " AND last_version = #{oldTagMeta.lastVersion}" + + " AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetadataObjectRelPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetadataObjectRelPostgreSQLProvider.java new file mode 100644 index 00000000000..1bb716248be --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetadataObjectRelPostgreSQLProvider.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelBaseSQLProvider; + +public class TagMetadataObjectRelPostgreSQLProvider extends TagMetadataObjectRelBaseSQLProvider { + @Override + public String softDeleteTagMetadataObjectRelsByMetalakeAndTagName( + String metalakeName, String tagName) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + @Override + public String softDeleteTagMetadataObjectRelsByMetalakeId(Long metalakeId) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE EXISTS (SELECT * FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + @Override + public String batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + Long metadataObjectId, String metadataObjectType, List tagIds) { + return ""; + } + + @Override + public String listTagMetadataObjectRelsByMetalakeAndTagName(String metalakeName, String tagName) { + return "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," + + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," + + " te.current_version as currentVersion, te.last_version as lastVersion," + + " te.deleted_at as deletedAt" + + " FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te JOIN " + + TagMetaMapper.TAG_TABLE_NAME + + " tm ON te.tag_id = tm.tag_id JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TopicMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TopicMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..ef7e45cbc1e --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TopicMetaPostgreSQLProvider.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.TopicMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.TopicPO; + +public class TopicMetaPostgreSQLProvider extends TopicMetaBaseSQLProvider { + + @Override + public String softDeleteTopicMetasByTopicId(Long topicId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE topic_id = #{topicId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTopicMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTopicMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTopicMetasBySchemaId(Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String insertTopicMetaOnDuplicateKeyUpdate(TopicPO topicPO) { + return "INSERT INTO " + + TABLE_NAME + + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," + + " comment, properties, audit_info, current_version, last_version," + + " deleted_at)" + + " VALUES(" + + " #{topicMeta.topicId}," + + " #{topicMeta.topicName}," + + " #{topicMeta.metalakeId}," + + " #{topicMeta.catalogId}," + + " #{topicMeta.schemaId}," + + " #{topicMeta.comment}," + + " #{topicMeta.properties}," + + " #{topicMeta.auditInfo}," + + " #{topicMeta.currentVersion}," + + " #{topicMeta.lastVersion}," + + " #{topicMeta.deletedAt}" + + " )" + + " ON CONFLICT (topic_id) DO UPDATE SET" + + " topic_name = #{topicMeta.topicName}," + + " metalake_id = #{topicMeta.metalakeId}," + + " catalog_id = #{topicMeta.catalogId}," + + " schema_id = #{topicMeta.schemaId}," + + " comment = #{topicMeta.comment}," + + " properties = #{topicMeta.properties}," + + " audit_info = #{topicMeta.auditInfo}," + + " current_version = #{topicMeta.currentVersion}," + + " last_version = #{topicMeta.lastVersion}," + + " deleted_at = #{topicMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..af7d65d2ab5 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserMetaPostgreSQLProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.UserMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.UserPO; + +public class UserMetaPostgreSQLProvider extends UserMetaBaseSQLProvider { + @Override + public String softDeleteUserMetaByUserId(Long userId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + @Override + public String softDeleteUserMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertUserMetaOnDuplicateKeyUpdate(UserPO userPO) { + return "INSERT INTO " + + USER_TABLE_NAME + + "(user_id, user_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )" + + " ON CONFLICT(user_id) DO UPDATE SET" + + " user_name = #{userMeta.userName}," + + " metalake_id = #{userMeta.metalakeId}," + + " audit_info = #{userMeta.auditInfo}," + + " current_version = #{userMeta.currentVersion}," + + " last_version = #{userMeta.lastVersion}," + + " deleted_at = #{userMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserRoleRelPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserRoleRelPostgreSQLProvider.java new file mode 100644 index 00000000000..dddca1efc17 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserRoleRelPostgreSQLProvider.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.mapper.UserRoleRelBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.UserRoleRelPO; + +public class UserRoleRelPostgreSQLProvider extends UserRoleRelBaseSQLProvider { + @Override + public String softDeleteUserRoleRelByUserId(Long userId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + @Override + public String softDeleteUserRoleRelByUserAndRoles(Long userId, List roleIds) { + return ""; + } + + @Override + public String softDeleteUserRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE user_id IN (SELECT user_id FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + @Override + public String softDeleteUserRoleRelByRoleId(Long roleId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + @Override + public String batchInsertUserRoleRelOnDuplicateKeyUpdate(List userRoleRelPOs) { + return ""; + } +} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java index b0ff97ea68d..6b2b147b1c8 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java @@ -116,7 +116,7 @@ public void createDatabase(TestDatabaseName testDatabaseName) { DriverManager.getConnection(getJdbcUrl(), getUsername(), getPassword()); Statement statement = connection.createStatement()) { - String query = format("CREATE DATABASE %s;", testDatabaseName); + String query = format("CREATE DATABASE \"%s\"", testDatabaseName); statement.execute(query); LOG.info(format("PostgreSQL container database %s has been created", testDatabaseName)); } catch (SQLException e) { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java index 968eeeef884..3e264652b31 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java @@ -19,6 +19,8 @@ package org.apache.gravitino.integration.test.util; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH; +import static org.apache.gravitino.integration.test.util.TestDatabaseName.PG_CATALOG_POSTGRESQL_IT; +import static org.apache.gravitino.integration.test.util.TestDatabaseName.PG_JDBC_BACKEND; import static org.apache.gravitino.server.GravitinoServer.WEBSERVER_CONF_PREFIX; import com.google.common.base.Splitter; @@ -30,6 +32,7 @@ import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; @@ -39,6 +42,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; @@ -49,6 +53,7 @@ import org.apache.gravitino.integration.test.MiniGravitinoContext; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.MySQLContainer; +import org.apache.gravitino.integration.test.container.PostgreSQLContainer; import org.apache.gravitino.server.GravitinoServer; import org.apache.gravitino.server.ServerConfig; import org.apache.gravitino.server.web.JettyServerConfig; @@ -92,6 +97,7 @@ public class AbstractIT { private static TestDatabaseName META_DATA; private static MySQLContainer MYSQL_CONTAINER; + private static PostgreSQLContainer POSTGRESQL_CONTAINER; protected static String serverUri; @@ -155,6 +161,63 @@ protected static void downLoadJDBCDriver() throws IOException { } } + protected static void setPGBackend() throws SQLException { + String pgUrlWithoutSchema = POSTGRESQL_CONTAINER.getJdbcUrl(META_DATA); + customConfigs.put(Configs.ENTITY_STORE_KEY, "relational"); + customConfigs.put(Configs.ENTITY_RELATIONAL_STORE_KEY, "JDBCBackend"); + customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY, pgUrlWithoutSchema); + customConfigs.put( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY, + POSTGRESQL_CONTAINER.getDriverClassName(META_DATA)); + customConfigs.put( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY, POSTGRESQL_CONTAINER.getUsername()); + customConfigs.put( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY, POSTGRESQL_CONTAINER.getPassword()); + + LOG.info("PG URL: {}", pgUrlWithoutSchema); + + String randomSchemaName = RandomStringUtils.random(10, true, false); + // Connect to the PostgreSQL docker and create a schema + String currentExecuteSql = ""; + try (Connection connection = + DriverManager.getConnection( + pgUrlWithoutSchema, + POSTGRESQL_CONTAINER.getUsername(), + POSTGRESQL_CONTAINER.getPassword())) { + connection.setCatalog(PG_CATALOG_POSTGRESQL_IT.toString()); + final Statement statement = connection.createStatement(); + statement.execute("drop schema if exists " + randomSchemaName); + statement.execute("create schema " + randomSchemaName); + statement.execute("set search_path to " + randomSchemaName); + String gravitinoHome = System.getenv("GRAVITINO_ROOT_DIR"); + String mysqlContent = + FileUtils.readFileToString( + new File( + gravitinoHome + + String.format( + "/scripts/postgresql/schema-%s-postgresql.sql", + ConfigConstants.VERSION_0_7_0)), + "UTF-8"); + + String[] initPGBackendSqls = + Arrays.stream(mysqlContent.split(";")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toArray(String[]::new); + + for (String sql : initPGBackendSqls) { + currentExecuteSql = sql; + statement.execute(sql); + } + } catch (Exception e) { + LOG.error("Failed to create database in pg, sql:\n{}", currentExecuteSql, e); + throw new RuntimeException(e); + } + + pgUrlWithoutSchema = pgUrlWithoutSchema + "?currentSchema=" + randomSchemaName; + customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY, pgUrlWithoutSchema); + } + private static void setMySQLBackend() { String mysqlUrl = MYSQL_CONTAINER.getJdbcUrl(META_DATA); customConfigs.put(Configs.ENTITY_STORE_KEY, "relational"); @@ -221,6 +284,13 @@ public static void startIntegrationTest() throws Exception { MYSQL_CONTAINER = containerSuite.getMySQLContainer(); setMySQLBackend(); + } else if ("PostgreSQL".equalsIgnoreCase(System.getenv("jdbcBackend"))) { + // Start PostgreSQL docker instance. + META_DATA = PG_JDBC_BACKEND; + containerSuite.startPostgreSQLContainer(META_DATA); + POSTGRESQL_CONTAINER = containerSuite.getPostgreSQLContainer(); + + setPGBackend(); } File baseDir = new File(System.getProperty("java.io.tmpdir")); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java index 98fb959ae77..4e81b992cc3 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java @@ -58,6 +58,8 @@ public enum TestDatabaseName { /** Represents the MySQL database used for testing the catalog integration with MySQL. */ MYSQL_CATALOG_MYSQL_IT, + PG_JDBC_BACKEND, + /** Represents the PostgreSQL database for CatalogPostgreSqlIT. */ PG_CATALOG_POSTGRESQL_IT { /** PostgreSQL only accept lowercase database name */ diff --git a/scripts/postgresql/schema-0.7.0-postgresql.sql b/scripts/postgresql/schema-0.7.0-postgresql.sql new file mode 100644 index 00000000000..0ce7a968537 --- /dev/null +++ b/scripts/postgresql/schema-0.7.0-postgresql.sql @@ -0,0 +1,458 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +-- Note: Database and schema creation is not included in this script. Please create the database and +-- schema before running this script. for example in psql: +-- CREATE DATABASE example_db; +-- \c example_db +-- CREATE SCHEMA example_schema; +-- set search_path to example_schema; + + +CREATE TABLE IF NOT EXISTS metalake_meta ( + metalake_id BIGINT NOT NULL, + metalake_name VARCHAR(128) NOT NULL, + metalake_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + schema_version TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (metalake_id), + UNIQUE (metalake_name, deleted_at) + ); +COMMENT ON TABLE metalake_meta IS 'metalake metadata'; + +COMMENT ON COLUMN metalake_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN metalake_meta.metalake_name IS 'metalake name'; +COMMENT ON COLUMN metalake_meta.metalake_comment IS 'metalake comment'; +COMMENT ON COLUMN metalake_meta.properties IS 'metalake properties'; +COMMENT ON COLUMN metalake_meta.audit_info IS 'metalake audit info'; +COMMENT ON COLUMN metalake_meta.schema_version IS 'metalake schema version info'; +COMMENT ON COLUMN metalake_meta.current_version IS 'metalake current version'; +COMMENT ON COLUMN metalake_meta.last_version IS 'metalake last version'; +COMMENT ON COLUMN metalake_meta.deleted_at IS 'metalake deleted at'; + + +CREATE TABLE IF NOT EXISTS catalog_meta ( + catalog_id BIGINT NOT NULL, + catalog_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + type VARCHAR(64) NOT NULL, + provider VARCHAR(64) NOT NULL, + catalog_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (catalog_id), + UNIQUE (metalake_id, catalog_name, deleted_at) + ); + +COMMENT ON TABLE catalog_meta IS 'catalog metadata'; + +COMMENT ON COLUMN catalog_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN catalog_meta.catalog_name IS 'catalog name'; +COMMENT ON COLUMN catalog_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN catalog_meta.type IS 'catalog type'; +COMMENT ON COLUMN catalog_meta.provider IS 'catalog provider'; +COMMENT ON COLUMN catalog_meta.catalog_comment IS 'catalog comment'; +COMMENT ON COLUMN catalog_meta.properties IS 'catalog properties'; +COMMENT ON COLUMN catalog_meta.audit_info IS 'catalog audit info'; +COMMENT ON COLUMN catalog_meta.current_version IS 'catalog current version'; +COMMENT ON COLUMN catalog_meta.last_version IS 'catalog last version'; +COMMENT ON COLUMN catalog_meta.deleted_at IS 'catalog deleted at'; + + +CREATE TABLE IF NOT EXISTS schema_meta ( + schema_id BIGINT NOT NULL, + schema_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (schema_id), + UNIQUE (catalog_id, schema_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON schema_meta (metalake_id); +COMMENT ON TABLE schema_meta IS 'schema metadata'; + +COMMENT ON COLUMN schema_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN schema_meta.schema_name IS 'schema name'; +COMMENT ON COLUMN schema_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN schema_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN schema_meta.schema_comment IS 'schema comment'; +COMMENT ON COLUMN schema_meta.properties IS 'schema properties'; +COMMENT ON COLUMN schema_meta.audit_info IS 'schema audit info'; +COMMENT ON COLUMN schema_meta.current_version IS 'schema current version'; +COMMENT ON COLUMN schema_meta.last_version IS 'schema last version'; +COMMENT ON COLUMN schema_meta.deleted_at IS 'schema deleted at'; + + +CREATE TABLE IF NOT EXISTS table_meta ( + table_id BIGINT NOT NULL, + table_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (table_id), + UNIQUE (schema_id, table_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON table_meta (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON table_meta (catalog_id); +COMMENT ON TABLE table_meta IS 'table metadata'; + +COMMENT ON COLUMN table_meta.table_id IS 'table id'; +COMMENT ON COLUMN table_meta.table_name IS 'table name'; +COMMENT ON COLUMN table_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN table_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN table_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN table_meta.audit_info IS 'table audit info'; +COMMENT ON COLUMN table_meta.current_version IS 'table current version'; +COMMENT ON COLUMN table_meta.last_version IS 'table last version'; +COMMENT ON COLUMN table_meta.deleted_at IS 'table deleted at'; + + +CREATE TABLE IF NOT EXISTS fileset_meta ( + fileset_id BIGINT NOT NULL, + fileset_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + type VARCHAR(64) NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (fileset_id), + UNIQUE (schema_id, fileset_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON fileset_meta (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON fileset_meta (catalog_id); +COMMENT ON TABLE fileset_meta IS 'fileset metadata'; + +COMMENT ON COLUMN fileset_meta.fileset_id IS 'fileset id'; +COMMENT ON COLUMN fileset_meta.fileset_name IS 'fileset name'; +COMMENT ON COLUMN fileset_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN fileset_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN fileset_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN fileset_meta.type IS 'fileset type'; +COMMENT ON COLUMN fileset_meta.audit_info IS 'fileset audit info'; +COMMENT ON COLUMN fileset_meta.current_version IS 'fileset current version'; +COMMENT ON COLUMN fileset_meta.last_version IS 'fileset last version'; +COMMENT ON COLUMN fileset_meta.deleted_at IS 'fileset deleted at'; + + +CREATE TABLE IF NOT EXISTS fileset_version_info ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + fileset_id BIGINT NOT NULL, + version INT NOT NULL, + fileset_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + storage_location TEXT NOT NULL, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (fileset_id, version, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON fileset_version_info (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON fileset_version_info (catalog_id); +CREATE INDEX IF NOT EXISTS idx_schema_id ON fileset_version_info (schema_id); +COMMENT ON TABLE fileset_version_info IS 'fileset version information'; + +COMMENT ON COLUMN fileset_version_info.id IS 'auto increment id'; +COMMENT ON COLUMN fileset_version_info.metalake_id IS 'metalake id'; +COMMENT ON COLUMN fileset_version_info.catalog_id IS 'catalog id'; +COMMENT ON COLUMN fileset_version_info.schema_id IS 'schema id'; +COMMENT ON COLUMN fileset_version_info.fileset_id IS 'fileset id'; +COMMENT ON COLUMN fileset_version_info.version IS 'fileset info version'; +COMMENT ON COLUMN fileset_version_info.fileset_comment IS 'fileset comment'; +COMMENT ON COLUMN fileset_version_info.properties IS 'fileset properties'; +COMMENT ON COLUMN fileset_version_info.storage_location IS 'fileset storage location'; +COMMENT ON COLUMN fileset_version_info.deleted_at IS 'fileset deleted at'; + + +CREATE TABLE IF NOT EXISTS topic_meta ( + topic_id BIGINT NOT NULL, + topic_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (topic_id), + UNIQUE (schema_id, topic_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON topic_meta (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON topic_meta (catalog_id); +COMMENT ON TABLE topic_meta IS 'topic metadata'; + +COMMENT ON COLUMN topic_meta.topic_id IS 'topic id'; +COMMENT ON COLUMN topic_meta.topic_name IS 'topic name'; +COMMENT ON COLUMN topic_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN topic_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN topic_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN topic_meta.comment IS 'topic comment'; +COMMENT ON COLUMN topic_meta.properties IS 'topic properties'; +COMMENT ON COLUMN topic_meta.audit_info IS 'topic audit info'; +COMMENT ON COLUMN topic_meta.current_version IS 'topic current version'; +COMMENT ON COLUMN topic_meta.last_version IS 'topic last version'; +COMMENT ON COLUMN topic_meta.deleted_at IS 'topic deleted at'; + + +CREATE TABLE IF NOT EXISTS user_meta ( + user_id BIGINT NOT NULL, + user_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (user_id), + UNIQUE (metalake_id, user_name, deleted_at) + ); +COMMENT ON TABLE user_meta IS 'user metadata'; + +COMMENT ON COLUMN user_meta.user_id IS 'user id'; +COMMENT ON COLUMN user_meta.user_name IS 'username'; +COMMENT ON COLUMN user_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN user_meta.audit_info IS 'user audit info'; +COMMENT ON COLUMN user_meta.current_version IS 'user current version'; +COMMENT ON COLUMN user_meta.last_version IS 'user last version'; +COMMENT ON COLUMN user_meta.deleted_at IS 'user deleted at'; + +CREATE TABLE IF NOT EXISTS role_meta ( + role_id BIGINT NOT NULL, + role_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (role_id), + UNIQUE (metalake_id, role_name, deleted_at) + ); + +COMMENT ON TABLE role_meta IS 'role metadata'; + +COMMENT ON COLUMN role_meta.role_id IS 'role id'; +COMMENT ON COLUMN role_meta.role_name IS 'role name'; +COMMENT ON COLUMN role_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN role_meta.properties IS 'role properties'; +COMMENT ON COLUMN role_meta.audit_info IS 'role audit info'; +COMMENT ON COLUMN role_meta.current_version IS 'role current version'; +COMMENT ON COLUMN role_meta.last_version IS 'role last version'; +COMMENT ON COLUMN role_meta.deleted_at IS 'role deleted at'; + + +CREATE TABLE IF NOT EXISTS role_meta_securable_object ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + role_id BIGINT NOT NULL, + metadata_object_id BIGINT NOT NULL, + type VARCHAR(128) NOT NULL, + privilege_names VARCHAR(256) NOT NULL, + privilege_conditions VARCHAR(256) NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id) + ); + +CREATE INDEX IF NOT EXISTS idx_role_id ON role_meta_securable_object (role_id); +COMMENT ON TABLE role_meta_securable_object IS 'role to securable object relation metadata'; + +COMMENT ON COLUMN role_meta_securable_object.id IS 'auto increment id'; +COMMENT ON COLUMN role_meta_securable_object.role_id IS 'role id'; +COMMENT ON COLUMN role_meta_securable_object.metadata_object_id IS 'The entity id of securable object'; +COMMENT ON COLUMN role_meta_securable_object.type IS 'securable object type'; +COMMENT ON COLUMN role_meta_securable_object.privilege_names IS 'securable object privilege names'; +COMMENT ON COLUMN role_meta_securable_object.privilege_conditions IS 'securable object privilege conditions'; +COMMENT ON COLUMN role_meta_securable_object.current_version IS 'securable object current version'; +COMMENT ON COLUMN role_meta_securable_object.last_version IS 'securable object last version'; +COMMENT ON COLUMN role_meta_securable_object.deleted_at IS 'securable object deleted at'; + + +CREATE TABLE IF NOT EXISTS user_role_rel ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + user_id BIGINT NOT NULL, + role_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (user_id, role_id, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_user_id ON user_role_rel (user_id); +COMMENT ON TABLE user_role_rel IS 'user role relation metadata'; + +COMMENT ON COLUMN user_role_rel.id IS 'auto increment id'; +COMMENT ON COLUMN user_role_rel.user_id IS 'user id'; +COMMENT ON COLUMN user_role_rel.role_id IS 'role id'; +COMMENT ON COLUMN user_role_rel.audit_info IS 'relation audit info'; +COMMENT ON COLUMN user_role_rel.current_version IS 'relation current version'; +COMMENT ON COLUMN user_role_rel.last_version IS 'relation last version'; +COMMENT ON COLUMN user_role_rel.deleted_at IS 'relation deleted at'; + + +CREATE TABLE IF NOT EXISTS group_meta ( + group_id BIGINT NOT NULL, + group_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (group_id), + UNIQUE (metalake_id, group_name, deleted_at) + ); +COMMENT ON TABLE group_meta IS 'group metadata'; + +COMMENT ON COLUMN group_meta.group_id IS 'group id'; +COMMENT ON COLUMN group_meta.group_name IS 'group name'; +COMMENT ON COLUMN group_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN group_meta.audit_info IS 'group audit info'; +COMMENT ON COLUMN group_meta.current_version IS 'group current version'; +COMMENT ON COLUMN group_meta.last_version IS 'group last version'; +COMMENT ON COLUMN group_meta.deleted_at IS 'group deleted at'; + + +CREATE TABLE IF NOT EXISTS group_role_rel ( + id BIGSERIAL NOT NULL, + group_id BIGINT NOT NULL, + role_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (group_id, role_id, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_group_id ON group_role_rel (group_id); +COMMENT ON TABLE group_role_rel IS 'relation between group and role'; +COMMENT ON COLUMN group_role_rel.id IS 'auto increment id'; +COMMENT ON COLUMN group_role_rel.group_id IS 'group id'; +COMMENT ON COLUMN group_role_rel.role_id IS 'role id'; +COMMENT ON COLUMN group_role_rel.audit_info IS 'relation audit info'; +COMMENT ON COLUMN group_role_rel.current_version IS 'relation current version'; +COMMENT ON COLUMN group_role_rel.last_version IS 'relation last version'; +COMMENT ON COLUMN group_role_rel.deleted_at IS 'relation deleted at'; + +CREATE TABLE IF NOT EXISTS tag_meta ( + tag_id BIGINT NOT NULL, + tag_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + tag_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (tag_id), + UNIQUE (metalake_id, tag_name, deleted_at) + ); + +COMMENT ON TABLE tag_meta IS 'tag metadata'; + +COMMENT ON COLUMN tag_meta.tag_id IS 'tag id'; +COMMENT ON COLUMN tag_meta.tag_name IS 'tag name'; +COMMENT ON COLUMN tag_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN tag_meta.tag_comment IS 'tag comment'; +COMMENT ON COLUMN tag_meta.properties IS 'tag properties'; +COMMENT ON COLUMN tag_meta.audit_info IS 'tag audit info'; + + +CREATE TABLE IF NOT EXISTS tag_relation_meta ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY, + tag_id BIGINT NOT NULL, + metadata_object_id BIGINT NOT NULL, + metadata_object_type VARCHAR(64) NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (tag_id, metadata_object_id, metadata_object_type, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_tag_id ON tag_relation_meta (tag_id); +CREATE INDEX IF NOT EXISTS idx_metadata_object_id ON tag_relation_meta (metadata_object_id); +COMMENT ON TABLE tag_relation_meta IS 'tag metadata object relation'; +COMMENT ON COLUMN tag_relation_meta.id IS 'auto increment id'; +COMMENT ON COLUMN tag_relation_meta.tag_id IS 'tag id'; +COMMENT ON COLUMN tag_relation_meta.metadata_object_id IS 'metadata object id'; +COMMENT ON COLUMN tag_relation_meta.metadata_object_type IS 'metadata object type'; +COMMENT ON COLUMN tag_relation_meta.audit_info IS 'tag relation audit info'; +COMMENT ON COLUMN tag_relation_meta.current_version IS 'tag relation current version'; +COMMENT ON COLUMN tag_relation_meta.last_version IS 'tag relation last version'; +COMMENT ON COLUMN tag_relation_meta.deleted_at IS 'tag relation deleted at'; + +CREATE TABLE IF NOT EXISTS owner_meta ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY, + metalake_id BIGINT NOT NULL, + owner_id BIGINT NOT NULL, + owner_type VARCHAR(64) NOT NULL, + metadata_object_id BIGINT NOT NULL, + metadata_object_type VARCHAR(64) NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (owner_id, metadata_object_id, metadata_object_type, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_owner_id ON owner_meta (owner_id); +CREATE INDEX IF NOT EXISTS idx_metadata_object_id ON owner_meta (metadata_object_id); +COMMENT ON TABLE owner_meta IS 'owner relation'; +COMMENT ON COLUMN owner_meta.id IS 'auto increment id'; +COMMENT ON COLUMN owner_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN owner_meta.owner_id IS 'owner id'; +COMMENT ON COLUMN owner_meta.owner_type IS 'owner type'; +COMMENT ON COLUMN owner_meta.metadata_object_id IS 'metadata object id'; +COMMENT ON COLUMN owner_meta.metadata_object_type IS 'metadata object type'; +COMMENT ON COLUMN owner_meta.audit_info IS 'owner relation audit info'; +COMMENT ON COLUMN owner_meta.current_version IS 'owner relation current version'; +COMMENT ON COLUMN owner_meta.last_version IS 'owner relation last version'; +COMMENT ON COLUMN owner_meta.deleted_at IS 'owner relation deleted at'; +