diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 1c6596f772a..c4c0a446759 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -61,12 +61,15 @@ jobs: # Integration test for AMD64 architecture architecture: [linux/amd64] java-version: [ 8, 11, 17 ] + backend: [ h2, mysql, postgresql ] test-mode: [ embedded, deploy ] - include: + exclude: - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' backend: 'mysql' + - test-mode: 'embedded' + backend: 'postgresql' + - test-mode: 'deploy' + backend: 'h2' env: PLATFORM: ${{ matrix.architecture }} @@ -132,11 +135,14 @@ jobs: architecture: [ linux/amd64 ] java-version: [ 17 ] test-mode: [ embedded, deploy ] - include: + backend: [ h2, mysql, postgresql ] + exclude: - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' backend: 'mysql' + - test-mode: 'embedded' + backend: 'postgresql' + - test-mode: 'deploy' + backend: 'h2' env: PLATFORM: ${{ matrix.architecture }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cf17683a0d..b934c4cc26a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,7 @@ jobs: - trino-connector/** - web/** - docs/open-api/** + - docs/build.gradle.kts - build.gradle.kts - gradle.properties - gradlew diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 75692e987b8..03e18bef914 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -194,8 +194,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n UserEntity.builder() .withId(1L) .withName(newOwner.name()) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); onUserAdded(userEntity); @@ -204,8 +203,7 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n GroupEntity.builder() .withId(1L) .withName(newOwner.name()) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); onGroupAdded(groupEntity); diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index 79f66ef287a..844b89b3008 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -483,8 +483,7 @@ public void testOnGrantedRolesToUser() { UserEntity.builder() .withId(1L) .withName(userName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -502,8 +501,7 @@ public void testOnGrantedRolesToUser() { UserEntity.builder() .withId(1L) .withName(userName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -525,8 +523,7 @@ public void testOnRevokedRolesFromUser() { UserEntity.builder() .withId(1L) .withName(userName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -555,8 +552,7 @@ public void testOnGrantedRolesToGroup() { GroupEntity.builder() .withId(1L) .withName(groupName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -574,8 +570,7 @@ public void testOnGrantedRolesToGroup() { GroupEntity.builder() .withId(1L) .withName(groupName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -598,8 +593,7 @@ public void testOnRevokedRolesFromGroup() { GroupEntity.builder() .withId(1L) .withName(groupName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -680,8 +674,7 @@ public void testCreateUser() { .withId(0L) .withName(currentFunName()) .withAuditInfo(auditInfo) - .withRoleIds(null) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); Assertions.assertTrue(rangerAuthPlugin.onUserAdded(user)); Assertions.assertTrue(rangerAuthPlugin.onUserAcquired(user)); @@ -696,8 +689,7 @@ public void testCreateGroup() { .withId(0L) .withName(currentFunName()) .withAuditInfo(auditInfo) - .withRoleIds(null) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); Assertions.assertTrue(rangerAuthPlugin.onGroupAdded(group)); @@ -769,8 +761,7 @@ public void testCombinationOperation() { UserEntity.builder() .withId(1L) .withName(userName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -786,8 +777,7 @@ public void testCombinationOperation() { UserEntity.builder() .withId(1L) .withName(userName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -800,8 +790,7 @@ public void testCombinationOperation() { UserEntity.builder() .withId(1L) .withName(userName3) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -833,8 +822,7 @@ public void testCombinationOperation() { GroupEntity.builder() .withId(1L) .withName(groupName1) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -852,8 +840,7 @@ public void testCombinationOperation() { GroupEntity.builder() .withId(1L) .withName(groupName2) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( @@ -871,8 +858,7 @@ public void testCombinationOperation() { GroupEntity.builder() .withId(1L) .withName(groupName3) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( diff --git a/build.gradle.kts b/build.gradle.kts index 3e47891f84f..cb7bad9a851 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -431,9 +431,9 @@ subprojects { reports.html.outputLocation.set(file("${rootProject.projectDir}/build/reports/")) val skipTests = project.hasProperty("skipTests") if (!skipTests) { - jvmArgs = listOf("-Xmx4G") + val extraArgs = project.property("extraJvmArgs") as List + jvmArgs = listOf("-Xmx4G") + extraArgs useJUnitPlatform() - jvmArgs(project.property("extraJvmArgs") as List<*>) finalizedBy(tasks.getByName("jacocoTestReport")) } } diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index ea38a895f08..429cd405243 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -79,6 +79,7 @@ dependencies { testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 1946464911f..0de7b27e159 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -69,7 +69,6 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; - private static final String SLASH = "/"; private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class); @@ -358,29 +357,7 @@ public boolean dropFileset(NameIdentifier ident) { @Override public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { - // TODO we need move some check logics in the Hadoop / Python GVFS to here. - Preconditions.checkArgument(subPath != null, "subPath must not be null"); - String processedSubPath; - if (!subPath.trim().isEmpty() && !subPath.trim().startsWith(SLASH)) { - processedSubPath = SLASH + subPath.trim(); - } else { - processedSubPath = subPath.trim(); - } - - Fileset fileset = loadFileset(ident); - - String fileLocation; - // subPath cannot be null, so we only need check if it is blank - if (StringUtils.isBlank(processedSubPath)) { - fileLocation = fileset.storageLocation(); - } else { - String storageLocation = - fileset.storageLocation().endsWith(SLASH) - ? fileset.storageLocation().substring(0, fileset.storageLocation().length() - 1) - : fileset.storageLocation(); - fileLocation = String.format("%s%s", storageLocation, processedSubPath); - } - return fileLocation; + throw new UnsupportedOperationException("Not implemented"); } @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java index 79777f75d64..284fd491d08 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java @@ -221,7 +221,7 @@ public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { @Override public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { - return hadoopCatalogOperations.getFileLocation(ident, subPath); + throw new UnsupportedOperationException("Not implemented"); } @Override diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 9f08ed8b685..3b9ca0d7f8b 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -762,46 +762,6 @@ public void testTestConnection() { ImmutableMap.of())); } - @Test - public void testGetFileLocation() throws IOException { - String schemaName = "schema1024"; - String comment = "comment1024"; - String schemaPath = TEST_ROOT_PATH + "/" + schemaName; - createSchema(schemaName, comment, null, schemaPath); - - String catalogName = "c1"; - String name = "fileset1024"; - String storageLocation = TEST_ROOT_PATH + "/" + catalogName + "/" + schemaName + "/" + name; - Fileset fileset = - createFileset(name, schemaName, comment, Fileset.Type.MANAGED, null, storageLocation); - - try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { - ops.initialize(Maps.newHashMap(), randomCatalogInfo(), HADOOP_PROPERTIES_METADATA); - NameIdentifier filesetIdent = NameIdentifier.of("m1", "c1", schemaName, name); - // test sub path starts with "/" - String subPath1 = "/test/test.parquet"; - String fileLocation1 = ops.getFileLocation(filesetIdent, subPath1); - Assertions.assertEquals( - String.format("%s%s", fileset.storageLocation(), subPath1), fileLocation1); - - // test sub path not starts with "/" - String subPath2 = "test/test.parquet"; - String fileLocation2 = ops.getFileLocation(filesetIdent, subPath2); - Assertions.assertEquals( - String.format("%s/%s", fileset.storageLocation(), subPath2), fileLocation2); - - // test sub path is null - String subPath3 = null; - Assertions.assertThrows( - IllegalArgumentException.class, () -> ops.getFileLocation(filesetIdent, subPath3)); - - // test sub path is blank but not null - String subPath4 = ""; - String fileLocation3 = ops.getFileLocation(filesetIdent, subPath4); - Assertions.assertEquals(fileset.storageLocation(), fileLocation3); - } - } - private static Stream locationArguments() { return Stream.of( // Honor the catalog location diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index 776e9bf39c9..081bfcbeb6e 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -132,6 +132,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.mockito.core) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") { exclude("org.apache.hadoop") diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts index b7cf6dfc871..ccfaaddfb7b 100644 --- a/catalogs/catalog-jdbc-doris/build.gradle.kts +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -52,6 +52,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) diff --git a/catalogs/catalog-jdbc-mysql/build.gradle.kts b/catalogs/catalog-jdbc-mysql/build.gradle.kts index 9ad6c739ab3..95f0578d3bf 100644 --- a/catalogs/catalog-jdbc-mysql/build.gradle.kts +++ b/catalogs/catalog-jdbc-mysql/build.gradle.kts @@ -55,6 +55,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts index a2493fa77d9..3165758a168 100644 --- a/catalogs/catalog-kafka/build.gradle.kts +++ b/catalogs/catalog-kafka/build.gradle.kts @@ -51,6 +51,7 @@ dependencies { testImplementation(libs.mockito.core) testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts index a6adf999d31..930e3e3354d 100644 --- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts +++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts @@ -78,6 +78,7 @@ dependencies { testImplementation(libs.slf4j.api) testImplementation(libs.junit.jupiter.api) testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) testImplementation(libs.bundles.log4j) testImplementation(libs.junit.jupiter.params) testImplementation(libs.testcontainers) diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java index 3e870e8d87e..e3b2aa9a6cf 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java @@ -20,9 +20,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -245,12 +245,13 @@ public String getFileLocation(NameIdentifier ident, String subPath) Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); CallerContext callerContext = CallerContext.CallerContextHolder.get(); - Map queryParams = Maps.newHashMap(); - queryParams.put("subPath", subPath); + + Map params = new HashMap<>(); + params.put("sub_path", RESTUtils.encodeString(subPath)); FileLocationResponse resp = restClient.get( - formatFilesetRequestPath(fullNamespace) + "/" + ident.name() + "/" + "fileLocation", - queryParams, + formatFileLocationRequestPath(fullNamespace, ident.name()), + params, FileLocationResponse.class, callerContext != null ? callerContext.context() : Collections.emptyMap(), ErrorHandlers.filesetErrorHandler()); @@ -270,6 +271,19 @@ static String formatFilesetRequestPath(Namespace ns) { .toString(); } + @VisibleForTesting + static String formatFileLocationRequestPath(Namespace ns, String name) { + Namespace schemaNs = Namespace.of(ns.level(0), ns.level(1)); + return new StringBuilder() + .append(formatSchemaRequestPath(schemaNs)) + .append("/") + .append(RESTUtils.encodeString(ns.level(2))) + .append("/filesets/") + .append(RESTUtils.encodeString(name)) + .append("/location") + .toString(); + } + /** * Check whether the namespace of a fileset is valid. * diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java index a3332449953..8a87fb72e96 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java @@ -22,12 +22,10 @@ import static org.apache.hc.core5.http.HttpStatus.SC_NOT_FOUND; import static org.apache.hc.core5.http.HttpStatus.SC_OK; import static org.apache.hc.core5.http.HttpStatus.SC_SERVER_ERROR; -import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.jsonwebtoken.lang.Maps; import java.nio.file.NoSuchFileException; import java.time.Instant; import java.util.HashMap; @@ -35,6 +33,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -61,11 +60,11 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.rest.RESTUtils; import org.apache.hc.core5.http.Method; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.platform.commons.util.StringUtils; import org.mockserver.matchers.Times; import org.mockserver.model.HttpRequest; import org.mockserver.model.HttpResponse; @@ -422,10 +421,11 @@ public void testGetFileLocation() throws JsonProcessingException { String mockSubPath = "mock_location/test"; String filesetPath = withSlash( - FilesetCatalog.formatFilesetRequestPath( - Namespace.of(metalakeName, catalogName, "schema1")) - + "/fileset1/fileLocation"); - Map queryParams = Maps.of("subPath", mockSubPath).build(); + FilesetCatalog.formatFileLocationRequestPath( + Namespace.of(metalakeName, catalogName, "schema1"), fileset.name())); + Map queryParams = new HashMap<>(); + queryParams.put("sub_path", RESTUtils.encodeString(mockSubPath)); + String mockFileLocation = String.format("file:/fileset/%s/%s/%s/%s", catalogName, "schema1", "fileset1", mockSubPath); FileLocationResponse resp = new FileLocationResponse(mockFileLocation); @@ -438,6 +438,42 @@ public void testGetFileLocation() throws JsonProcessingException { NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath); Assertions.assertTrue(StringUtils.isNotBlank(actualFileLocation)); Assertions.assertEquals(mockFileLocation, actualFileLocation); + + // Throw schema not found exception + ErrorResponse errResp = + ErrorResponse.notFound(NoSuchSchemaException.class.getSimpleName(), "schema not found"); + buildMockResource(Method.GET, filesetPath, null, errResp, SC_NOT_FOUND); + Assertions.assertThrows( + NoSuchSchemaException.class, + () -> + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath), + "schema not found"); + + ErrorResponse errResp1 = + ErrorResponse.notFound(NotFoundException.class.getSimpleName(), "fileset not found"); + buildMockResource(Method.GET, filesetPath, null, errResp1, SC_NOT_FOUND); + Assertions.assertThrows( + NotFoundException.class, + () -> + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath), + "fileset not found"); + + ErrorResponse errResp2 = ErrorResponse.internalError("internal error"); + buildMockResource(Method.GET, filesetPath, null, errResp2, SC_SERVER_ERROR); + Assertions.assertThrows( + RuntimeException.class, + () -> + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath), + "internal error"); } @Test @@ -446,10 +482,10 @@ public void testCallerContextToHeader() throws JsonProcessingException { String mockSubPath = "mock_location/test"; String filesetPath = withSlash( - FilesetCatalog.formatFilesetRequestPath( - Namespace.of(metalakeName, catalogName, "schema1")) - + "/fileset1/fileLocation"); - Map queryParams = Maps.of("subPath", mockSubPath).build(); + FilesetCatalog.formatFileLocationRequestPath( + Namespace.of(metalakeName, catalogName, "schema1"), fileset.name())); + Map queryParams = new HashMap<>(); + queryParams.put("sub_path", RESTUtils.encodeString(mockSubPath)); String mockFileLocation = String.format("file:/fileset/%s/%s/%s/%s", catalogName, "schema1", "fileset1", mockSubPath); FileLocationResponse resp = new FileLocationResponse(mockFileLocation); @@ -496,8 +532,8 @@ public void testCallerContextToHeader() throws JsonProcessingException { .asFilesetCatalog() .getFileLocation( NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath); - assertEquals(FilesetDataOperation.GET_FILE_STATUS.name(), dataOperation.get()); - assertEquals(InternalClientType.HADOOP_GVFS.name(), internalClientType.get()); + Assertions.assertEquals(FilesetDataOperation.GET_FILE_STATUS.name(), dataOperation.get()); + Assertions.assertEquals(InternalClientType.HADOOP_GVFS.name(), internalClientType.get()); } private FilesetDTO mockFilesetDTO( diff --git a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java index edb02cdcec5..479bb069e3f 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java @@ -22,9 +22,11 @@ import static org.apache.gravitino.authorization.AuthorizationUtils.USER_DOES_NOT_EXIST_MSG; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -70,14 +72,10 @@ User grantRolesToUser(String metalake, List roles, String user) { UserEntity.class, Entity.EntityType.USER, userEntity -> { - List roleEntities = Lists.newArrayList(); - if (userEntity.roleNames() != null) { - for (String role : userEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } - } - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); - List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + List roleEntities = Lists.newArrayList(userEntity.roleEntities()); + + List roleIds = + roleEntities.stream().map(RoleEntity::id).collect(Collectors.toList()); for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { if (roleIds.contains(roleEntityToGrant.id())) { @@ -87,8 +85,8 @@ User grantRolesToUser(String metalake, List roles, String user) { user, metalake); } else { - roleNames.add(roleEntityToGrant.name()); roleIds.add(roleEntityToGrant.id()); + roleEntities.add(roleEntityToGrant); } } @@ -104,8 +102,7 @@ User grantRolesToUser(String metalake, List roles, String user) { .withNamespace(userEntity.namespace()) .withId(userEntity.id()) .withName(userEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(auditInfo) .build(); }); @@ -149,13 +146,8 @@ Group grantRolesToGroup(String metalake, List roles, String group) { GroupEntity.class, Entity.EntityType.GROUP, groupEntity -> { - List roleEntities = Lists.newArrayList(); - if (groupEntity.roleNames() != null) { - for (String role : groupEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } - } - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); + List roleEntities = Lists.newArrayList(groupEntity.roleEntities()); + List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { @@ -166,8 +158,8 @@ Group grantRolesToGroup(String metalake, List roles, String group) { group, metalake); } else { - roleNames.add(roleEntityToGrant.name()); roleIds.add(roleEntityToGrant.id()); + roleEntities.add(roleEntityToGrant); } } @@ -183,8 +175,7 @@ Group grantRolesToGroup(String metalake, List roles, String group) { .withId(groupEntity.id()) .withNamespace(groupEntity.namespace()) .withName(groupEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(auditInfo) .build(); }); @@ -228,19 +219,18 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) { GroupEntity.class, Entity.EntityType.GROUP, groupEntity -> { - List roleEntities = Lists.newArrayList(); - if (groupEntity.roleNames() != null) { - for (String role : groupEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } + List roleEntities = groupEntity.roleEntities(); + + // Avoid loading securable objects + Map roleEntitiesMap = Maps.newHashMap(); + for (RoleEntity entity : roleEntities) { + roleEntitiesMap.put(entity.id(), entity); } - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); - List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); for (RoleEntity roleEntityToRevoke : roleEntitiesToRevoke) { - roleNames.remove(roleEntityToRevoke.name()); - boolean removed = roleIds.remove(roleEntityToRevoke.id()); - if (!removed) { + if (roleEntitiesMap.containsKey(roleEntityToRevoke.id())) { + roleEntitiesMap.remove(roleEntityToRevoke.id()); + } else { LOG.warn( "Failed to revoke, role {} does not exist in the group {} of metalake {}", roleEntityToRevoke.name(), @@ -261,8 +251,7 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) { .withNamespace(groupEntity.namespace()) .withId(groupEntity.id()) .withName(groupEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(Lists.newArrayList(roleEntitiesMap.values())) .withAuditInfo(auditInfo) .build(); }); @@ -308,20 +297,18 @@ User revokeRolesFromUser(String metalake, List roles, String user) { UserEntity.class, Entity.EntityType.USER, userEntity -> { - List roleEntities = Lists.newArrayList(); - if (userEntity.roleNames() != null) { - for (String role : userEntity.roleNames()) { - roleEntities.add(roleManager.getRole(metalake, role)); - } - } + List roleEntities = userEntity.roleEntities(); - List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); - List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); + // Avoid loading securable objects + Map roleEntitiesMap = Maps.newHashMap(); + for (RoleEntity entity : roleEntities) { + roleEntitiesMap.put(entity.id(), entity); + } for (RoleEntity roleEntityToRevoke : roleEntitiesToRevoke) { - roleNames.remove(roleEntityToRevoke.name()); - boolean removed = roleIds.remove(roleEntityToRevoke.id()); - if (!removed) { + if (roleEntitiesMap.containsKey(roleEntityToRevoke.id())) { + roleEntitiesMap.remove(roleEntityToRevoke.id()); + } else { LOG.warn( "Failed to revoke, role {} doesn't exist in the user {} of metalake {}", roleEntityToRevoke.name(), @@ -341,8 +328,7 @@ User revokeRolesFromUser(String metalake, List roles, String user) { .withId(userEntity.id()) .withNamespace(userEntity.namespace()) .withName(userEntity.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(Lists.newArrayList(roleEntitiesMap.values())) .withAuditInfo(auditInfo) .build(); }); @@ -373,10 +359,6 @@ User revokeRolesFromUser(String metalake, List roles, String user) { } } - private List toRoleNames(List roleEntities) { - return roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList()); - } - private List toRoleIds(List roleEntities) { return roleEntities.stream().map(RoleEntity::id).collect(Collectors.toList()); } diff --git a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java index 09427668969..eef947f22e3 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java @@ -63,7 +63,7 @@ User addUser(String metalake, String name) throws UserAlreadyExistsException { .withId(idGenerator.nextId()) .withName(name) .withNamespace(AuthorizationUtils.ofUserNamespace(metalake)) - .withRoleNames(Lists.newArrayList()) + .withRoles(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) @@ -117,7 +117,7 @@ Group addGroup(String metalake, String group) throws GroupAlreadyExistsException .withId(idGenerator.nextId()) .withName(group) .withNamespace(AuthorizationUtils.ofGroupNamespace(metalake)) - .withRoleNames(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo( AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) diff --git a/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java b/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java index d9e4b633d54..047c2bcce99 100644 --- a/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/GroupEntity.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; import org.apache.gravitino.Field; @@ -38,11 +40,8 @@ public class GroupEntity implements Group, Entity, Auditable, HasIdentifier { public static final Field NAME = Field.required("name", String.class, "The name of the group entity."); - public static final Field ROLE_NAMES = - Field.optional("role_names", List.class, "The role names of the group entity."); - - public static final Field ROLE_IDS = - Field.optional("role_ids", List.class, "The role names of the group entity."); + public static final Field ROLES_SUPPLIER = + Field.required("roles_supplier", Supplier.class, "The roles supplier of the group entity."); public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the group entity."); @@ -50,9 +49,9 @@ public class GroupEntity implements Group, Entity, Auditable, HasIdentifier { private Long id; private String name; private AuditInfo auditInfo; - private List roleNames; - private List roleIds; private Namespace namespace; + // The roles is a lazy field to avoid unnecessary cost. + private Supplier> rolesSupplier = Collections::emptyList; private GroupEntity() {} @@ -67,8 +66,7 @@ public Map fields() { fields.put(ID, id); fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); - fields.put(ROLE_NAMES, roleNames); - fields.put(ROLE_IDS, roleIds); + fields.put(ROLES_SUPPLIER, rolesSupplier); return Collections.unmodifiableMap(fields); } @@ -130,25 +128,20 @@ public AuditInfo auditInfo() { */ @Override public List roles() { - return roleNames; - } + if (roleEntities() == null) { + return null; + } - /** - * Returns the role names of the group entity. - * - * @return The role names of the group entity. - */ - public List roleNames() { - return roleNames; + return roleEntities().stream().map(RoleEntity::name).collect(Collectors.toList()); } /** - * Returns the role ids of the group entity. + * Returns the role entities of the group entity. * - * @return The role ids of the group entity. + * @return The role entities of the group entity. */ - public List roleIds() { - return roleIds; + public List roleEntities() { + return rolesSupplier.get(); } @Override @@ -161,13 +154,12 @@ public boolean equals(Object o) { && Objects.equals(name, that.name) && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) - && Objects.equals(roleNames, that.roleNames) - && Objects.equals(roleIds, that.roleIds); + && Objects.equals(roles(), that.roles()); } @Override public int hashCode() { - return Objects.hash(id, name, auditInfo, roleNames, roleIds); + return Objects.hash(id, name, auditInfo, roles()); } public static Builder builder() { @@ -215,24 +207,26 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the role names of the group entity. + * Sets the roles of the group entity. * * @param roles The role names of the group entity. * @return The builder instance. */ - public Builder withRoleNames(List roles) { - groupEntity.roleNames = roles; + public Builder withRoles(List roles) { + if (roles != null) { + groupEntity.rolesSupplier = () -> roles; + } return this; } /** - * Sets the role ids of the group entity. + * Sets the roles supplier of the group entity. * - * @param roleIds The role ids of the group entity. + * @param supplier The roles supplier of the group entity. * @return The builder instance. */ - public Builder withRoleIds(List roleIds) { - groupEntity.roleIds = roleIds; + public Builder withRolesSupplier(Supplier> supplier) { + groupEntity.rolesSupplier = supplier; return this; } diff --git a/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java b/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java index d1b3bbfe99f..eda0bc89903 100644 --- a/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/RoleEntity.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; import org.apache.gravitino.Field; @@ -46,15 +47,18 @@ public class RoleEntity implements Role, Entity, Auditable, HasIdentifier { public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the role entity."); - public static final Field SECURABLE_OBJECT = - Field.required("securable_objects", List.class, "The securable objects of the role entity."); + public static final Field SECURABLE_OBJECTS_SUPPLIER = + Field.required( + "securable_objects_supplier", + Supplier.class, + "The securable objects supplier of the role entity."); private Long id; private String name; private Map properties; private AuditInfo auditInfo; private Namespace namespace; - private List securableObjects; + private Supplier> securableObjectsSupplier = () -> null; /** * The name of the role. @@ -91,16 +95,7 @@ public Map properties() { */ @Override public List securableObjects() { - // The securable object is a special kind of entities. Some entity types aren't the securable - // object, such as - // User, Role, etc. - // The securable object identifier must be unique. - // Gravitino assumes that the identifiers of the entities may be the same if they have different - // types. - // So one type of them can't be the securable object at least if there are the two same - // identifier - // entities . - return securableObjects; + return securableObjectsSupplier.get(); } /** @@ -115,7 +110,7 @@ public Map fields() { fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); fields.put(PROPERTIES, properties); - fields.put(SECURABLE_OBJECT, securableObjects); + fields.put(SECURABLE_OBJECTS_SUPPLIER, securableObjectsSupplier); return Collections.unmodifiableMap(fields); } @@ -151,12 +146,12 @@ public boolean equals(Object o) { && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties) - && Objects.equals(securableObjects, that.securableObjects); + && Objects.equals(securableObjects(), that.securableObjects()); } @Override public int hashCode() { - return Objects.hash(id, name, properties, auditInfo, securableObjects, namespace); + return Objects.hash(id, name, properties, auditInfo, securableObjects(), namespace); } /** @@ -231,7 +226,19 @@ public Builder withAuditInfo(AuditInfo auditInfo) { * @return The builder instance. */ public Builder withSecurableObjects(List securableObjects) { - roleEntity.securableObjects = ImmutableList.copyOf(securableObjects); + roleEntity.securableObjectsSupplier = () -> ImmutableList.copyOf(securableObjects); + return this; + } + + /** + * Sets the securable objects supplier of the role entity. + * + * @param securableObjectsSupplier The securable objects supplier of the role entity. + * @return The builder instance. + */ + public Builder withSecurableObjectsSupplier( + Supplier> securableObjectsSupplier) { + roleEntity.securableObjectsSupplier = securableObjectsSupplier; return this; } diff --git a/core/src/main/java/org/apache/gravitino/meta/UserEntity.java b/core/src/main/java/org/apache/gravitino/meta/UserEntity.java index c71d731a99e..7e12bd2d64a 100644 --- a/core/src/main/java/org/apache/gravitino/meta/UserEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/UserEntity.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; import lombok.ToString; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; @@ -44,18 +46,15 @@ public class UserEntity implements User, Entity, Auditable, HasIdentifier { public static final Field AUDIT_INFO = Field.required("audit_info", AuditInfo.class, "The audit details of the user entity."); - public static final Field ROLE_NAMES = - Field.optional("role_names", List.class, "The role names of the user entity"); - - public static final Field ROLE_IDS = - Field.optional("role_ids", List.class, "The role ids of the user entity"); + public static final Field ROLES_SUPPLIER = + Field.required("roles_supplier", Supplier.class, "The roles supplier of the user entity"); private Long id; private String name; private AuditInfo auditInfo; - private List roleNames; - private List roleIds; private Namespace namespace; + // The roles is a lazy field to avoid unnecessary cost. + private Supplier> rolesSupplier = Collections::emptyList; private UserEntity() {} @@ -70,8 +69,7 @@ public Map fields() { fields.put(ID, id); fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); - fields.put(ROLE_NAMES, roleNames); - fields.put(ROLE_IDS, roleIds); + fields.put(ROLES_SUPPLIER, rolesSupplier); return Collections.unmodifiableMap(fields); } @@ -133,25 +131,16 @@ public AuditInfo auditInfo() { */ @Override public List roles() { - return roleNames; - } - - /** - * Returns the role names of the user entity. - * - * @return The role names of the user entity. - */ - public List roleNames() { - return roleNames; + return roleEntities().stream().map(RoleEntity::name).collect(Collectors.toList()); } /** - * Returns the role ids of the user entity. + * Returns the role entities of the user entity. * - * @return The role ids of the user entity. + * @return The role entities of the user entity. */ - public List roleIds() { - return roleIds; + public List roleEntities() { + return rolesSupplier.get(); } @Override @@ -164,13 +153,12 @@ public boolean equals(Object o) { && Objects.equals(name, that.name) && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) - && Objects.equals(roleNames, that.roleNames) - && Objects.equals(roleIds, that.roleIds); + && Objects.equals(roles(), that.roles()); } @Override public int hashCode() { - return Objects.hash(id, name, auditInfo, roleNames, roleIds); + return Objects.hash(id, name, auditInfo, roles()); } public static Builder builder() { @@ -218,24 +206,26 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the role names of the user entity. + * Sets the roles of the user entity. * - * @param roles The role names of the user entity. + * @param roles The role entities of the user entity. * @return The builder instance. */ - public Builder withRoleNames(List roles) { - userEntity.roleNames = roles; + public Builder withRoles(List roles) { + if (roles != null) { + userEntity.rolesSupplier = () -> roles; + } return this; } /** - * Sets the role ids of the user entity. + * Sets the roles supplier of the user entity. * - * @param roleIds The role ids of the user entity. + * @param supplier The roles supplier of the user entity. * @return The builder instance. */ - public Builder withRoleIds(List roleIds) { - userEntity.roleIds = roleIds; + public Builder withRolesSupplier(Supplier> supplier) { + userEntity.rolesSupplier = supplier; return this; } diff --git a/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java deleted file mode 100644 index 46eb9c90841..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/GroupEntitySerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import java.util.Collection; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.GroupEntity; - -public class GroupEntitySerDe implements ProtoSerDe { - - @Override - public Group serialize(GroupEntity groupEntity) { - Group.Builder builder = - Group.newBuilder() - .setId(groupEntity.id()) - .setName(groupEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(groupEntity.auditInfo())); - - if (isCollectionNotEmpty(groupEntity.roles())) { - builder.addAllRoleNames(groupEntity.roles()); - } - - if (isCollectionNotEmpty(groupEntity.roleIds())) { - builder.addAllRoleIds(groupEntity.roleIds()); - } - - return builder.build(); - } - - @Override - public GroupEntity deserialize(Group group, Namespace namespace) { - GroupEntity.Builder builder = - GroupEntity.builder() - .withId(group.getId()) - .withName(group.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo(), namespace)); - - if (group.getRoleNamesCount() > 0) { - builder.withRoleNames(group.getRoleNamesList()); - } - - if (group.getRoleIdsCount() > 0) { - builder.withRoleIds(group.getRoleIdsList()); - } - - return builder.build(); - } - - private boolean isCollectionNotEmpty(Collection collection) { - return collection != null && !collection.isEmpty(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java index 3dc5cb1c297..a4b01a896d8 100644 --- a/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/org/apache/gravitino/proto/ProtoEntitySerDe.java @@ -53,11 +53,6 @@ public class ProtoEntitySerDe implements EntitySerDe { .put( "org.apache.gravitino.meta.TopicEntity", "org.apache.gravitino.proto.TopicEntitySerDe") - .put("org.apache.gravitino.meta.UserEntity", "org.apache.gravitino.proto.UserEntitySerDe") - .put( - "org.apache.gravitino.meta.GroupEntity", - "org.apache.gravitino.proto.GroupEntitySerDe") - .put("org.apache.gravitino.meta.RoleEntity", "org.apache.gravitino.proto.RoleEntitySerDe") .build(); private static final Map ENTITY_TO_PROTO = @@ -75,13 +70,7 @@ public class ProtoEntitySerDe implements EntitySerDe { "org.apache.gravitino.meta.FilesetEntity", "org.apache.gravitino.proto.Fileset", "org.apache.gravitino.meta.TopicEntity", - "org.apache.gravitino.proto.Topic", - "org.apache.gravitino.meta.UserEntity", - "org.apache.gravitino.proto.User", - "org.apache.gravitino.meta.GroupEntity", - "org.apache.gravitino.proto.Group", - "org.apache.gravitino.meta.RoleEntity", - "org.apache.gravitino.proto.Role"); + "org.apache.gravitino.proto.Topic"); private final Map, ProtoSerDe> entityToSerDe; diff --git a/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java deleted file mode 100644 index a54f4505d7f..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/RoleEntitySerDe.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import com.google.common.collect.Lists; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.gravitino.MetadataObject; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.authorization.Privilege; -import org.apache.gravitino.authorization.Privileges; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.meta.RoleEntity; - -public class RoleEntitySerDe implements ProtoSerDe { - - /** - * Serializes the provided entity into its corresponding Protocol Buffer message representation. - * - * @param roleEntity The entity to be serialized. - * @return The Protocol Buffer message representing the serialized entity. - */ - @Override - public Role serialize(RoleEntity roleEntity) { - Role.Builder builder = - Role.newBuilder() - .setId(roleEntity.id()) - .setName(roleEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())); - - for (SecurableObject securableObject : roleEntity.securableObjects()) { - builder.addSecurableObjects( - org.apache.gravitino.proto.SecurableObject.newBuilder() - .setFullName(securableObject.fullName()) - .setType(securableObject.type().name()) - .addAllPrivilegeConditions( - securableObject.privileges().stream() - .map(Privilege::condition) - .map(Privilege.Condition::name) - .collect(Collectors.toList())) - .addAllPrivilegeNames( - securableObject.privileges().stream() - .map(Privilege::name) - .map(Privilege.Name::name) - .collect(Collectors.toList())) - .build()); - } - - if (roleEntity.properties() != null && !roleEntity.properties().isEmpty()) { - builder.putAllProperties(roleEntity.properties()); - } - - return builder.build(); - } - - /** - * Deserializes the provided Protocol Buffer message into its corresponding entity representation. - * - * @param role The Protocol Buffer message to be deserialized. - * @return The entity representing the deserialized Protocol Buffer message. - */ - @Override - public RoleEntity deserialize(Role role, Namespace namespace) { - List securableObjects = Lists.newArrayList(); - - for (int index = 0; index < role.getSecurableObjectsCount(); index++) { - List privileges = Lists.newArrayList(); - org.apache.gravitino.proto.SecurableObject object = role.getSecurableObjects(index); - for (int privIndex = 0; privIndex < object.getPrivilegeConditionsCount(); privIndex++) { - if (Privilege.Condition.ALLOW.name().equals(object.getPrivilegeConditions(privIndex))) { - privileges.add(Privileges.allow(object.getPrivilegeNames(privIndex))); - } else { - privileges.add(Privileges.deny(object.getPrivilegeNames(privIndex))); - } - } - - SecurableObject securableObject = - SecurableObjects.parse( - object.getFullName(), MetadataObject.Type.valueOf(object.getType()), privileges); - - securableObjects.add(securableObject); - } - - RoleEntity.Builder builder = - RoleEntity.builder() - .withId(role.getId()) - .withName(role.getName()) - .withNamespace(namespace) - .withSecurableObjects(securableObjects) - .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); - - if (!role.getPropertiesMap().isEmpty()) { - builder.withProperties(role.getPropertiesMap()); - } - - return builder.build(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java b/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java deleted file mode 100644 index e859ee3b881..00000000000 --- a/core/src/main/java/org/apache/gravitino/proto/UserEntitySerDe.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.proto; - -import java.util.Collection; -import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.UserEntity; - -public class UserEntitySerDe implements ProtoSerDe { - - @Override - public User serialize(UserEntity userEntity) { - User.Builder builder = - User.newBuilder() - .setId(userEntity.id()) - .setName(userEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(userEntity.auditInfo())); - - if (isCollectionNotEmpty(userEntity.roles())) { - builder.addAllRoleNames(userEntity.roles()); - } - - if (isCollectionNotEmpty(userEntity.roleIds())) { - builder.addAllRoleIds(userEntity.roleIds()); - } - - return builder.build(); - } - - @Override - public UserEntity deserialize(User user, Namespace namespace) { - UserEntity.Builder builder = - UserEntity.builder() - .withId(user.getId()) - .withName(user.getName()) - .withNamespace(namespace) - .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo(), namespace)); - - if (user.getRoleNamesCount() > 0) { - builder.withRoleNames(user.getRoleNamesList()); - } - - if (user.getRoleIdsCount() > 0) { - builder.withRoleIds(user.getRoleIdsList()); - } - - return builder.build(); - } - - private boolean isCollectionNotEmpty(Collection collection) { - return collection != null && !collection.isEmpty(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java b/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java index fa6fb48c636..2dd0e00b0cf 100644 --- a/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java +++ b/core/src/main/java/org/apache/gravitino/storage/kv/BinaryEntityKeyEncoder.java @@ -20,13 +20,10 @@ import static org.apache.gravitino.Entity.EntityType.CATALOG; import static org.apache.gravitino.Entity.EntityType.FILESET; -import static org.apache.gravitino.Entity.EntityType.GROUP; import static org.apache.gravitino.Entity.EntityType.METALAKE; -import static org.apache.gravitino.Entity.EntityType.ROLE; import static org.apache.gravitino.Entity.EntityType.SCHEMA; import static org.apache.gravitino.Entity.EntityType.TABLE; import static org.apache.gravitino.Entity.EntityType.TOPIC; -import static org.apache.gravitino.Entity.EntityType.USER; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -93,12 +90,6 @@ public class BinaryEntityKeyEncoder implements EntityKeyEncoder { new String[] {TABLE.getShortName() + "/", "/", "/", "/"}, FILESET, new String[] {FILESET.getShortName() + "/", "/", "/", "/"}, - USER, - new String[] {USER.getShortName() + "/", "/", "/", "/"}, - GROUP, - new String[] {GROUP.getShortName() + "/", "/", "/", "/"}, - ROLE, - new String[] {ROLE.getShortName() + "/", "/", "/", "/"}, TOPIC, new String[] {TOPIC.getShortName() + "/", "/", "/", "/"}); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 81e111a289b..b23c7667388 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -417,9 +417,10 @@ public void insertRelation( } } - enum JDBCBackendType { + public enum JDBCBackendType { H2(true), - MYSQL(false); + MYSQL(false), + POSTGRESQL(false); private final boolean embedded; @@ -432,10 +433,25 @@ public static JDBCBackendType fromURI(String jdbcURI) { return JDBCBackendType.H2; } else if (jdbcURI.startsWith("jdbc:mysql")) { return JDBCBackendType.MYSQL; + } else if (jdbcURI.startsWith("jdbc:postgresql")) { + return JDBCBackendType.POSTGRESQL; } else { throw new IllegalArgumentException("Unknown JDBC URI: " + jdbcURI); } } + + public static JDBCBackendType fromString(String jdbcType) { + switch (jdbcType) { + case "h2": + return JDBCBackendType.H2; + case "mysql": + return JDBCBackendType.MYSQL; + case "postgresql": + return JDBCBackendType.POSTGRESQL; + default: + throw new IllegalArgumentException("Unknown JDBC type: " + jdbcType); + } + } } /** Start JDBC database if necessary. For example, start the H2 database if the backend is H2. */ diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/converters/PostgreSQLExceptionConverter.java b/core/src/main/java/org/apache/gravitino/storage/relational/converters/PostgreSQLExceptionConverter.java new file mode 100644 index 00000000000..7414760930d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/converters/PostgreSQLExceptionConverter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.converters; + +import java.io.IOException; +import java.sql.SQLException; +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; + +/** + * Exception converter to Apache Gravitino exception for PostgreSQL. The definition of error codes + * can be found in the document: error code of PostgreSQL + */ +public class PostgreSQLExceptionConverter implements SQLExceptionConverter { + private static final int DUPLICATED_ENTRY_ERROR_CODE = 23505; + + @Override + @SuppressWarnings("FormatStringAnnotation") + public void toGravitinoException(SQLException sqlException, Entity.EntityType type, String name) + throws IOException { + int errorCode = Integer.valueOf(sqlException.getSQLState()); + switch (errorCode) { + case DUPLICATED_ENTRY_ERROR_CODE: + throw new EntityAlreadyExistsException( + sqlException, "The %s entity: %s already exists.", type.name(), name); + default: + throw new IOException(sqlException); + } + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java index 4954910c9ec..feb0fb0d7df 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/converters/SQLExceptionConverterFactory.java @@ -40,6 +40,8 @@ public static synchronized void initConverter(Config config) { converter = new MySQLExceptionConverter(); } else if (jdbcType.equalsIgnoreCase("h2")) { converter = new H2ExceptionConverter(); + } else if (jdbcType.equalsIgnoreCase("postgresql")) { + converter = new PostgreSQLExceptionConverter(); } else { throw new IllegalArgumentException(String.format("Unsupported jdbc type: %s", jdbcType)); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaBaseSQLProvider.java new file mode 100644 index 00000000000..cce5783898f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaBaseSQLProvider.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.CatalogPO; +import org.apache.ibatis.annotations.Param; + +public class CatalogMetaBaseSQLProvider { + public String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String selectCatalogIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return "SELECT catalog_id as catalogId FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0"; + } + + public String selectCatalogMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0"; + } + + public String selectCatalogMetaById(@Param("catalogId") Long catalogId) { + return "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO) { + return "INSERT INTO " + + TABLE_NAME + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{catalogMeta.catalogId}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + + " )"; + } + + public String insertCatalogMetaOnDuplicateKeyUpdate(@Param("catalogMeta") CatalogPO catalogPO) { + return "INSERT INTO " + + TABLE_NAME + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{catalogMeta.catalogId}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " catalog_name = #{catalogMeta.catalogName}," + + " metalake_id = #{catalogMeta.metalakeId}," + + " type = #{catalogMeta.type}," + + " provider = #{catalogMeta.provider}," + + " catalog_comment = #{catalogMeta.catalogComment}," + + " properties = #{catalogMeta.properties}," + + " audit_info = #{catalogMeta.auditInfo}," + + " current_version = #{catalogMeta.currentVersion}," + + " last_version = #{catalogMeta.lastVersion}," + + " deleted_at = #{catalogMeta.deletedAt}"; + } + + public String updateCatalogMeta( + @Param("newCatalogMeta") CatalogPO newCatalogPO, + @Param("oldCatalogMeta") CatalogPO oldCatalogPO) { + return "UPDATE " + + TABLE_NAME + + " SET catalog_name = #{newCatalogMeta.catalogName}," + + " metalake_id = #{newCatalogMeta.metalakeId}," + + " type = #{newCatalogMeta.type}," + + " provider = #{newCatalogMeta.provider}," + + " catalog_comment = #{newCatalogMeta.catalogComment}," + + " properties = #{newCatalogMeta.properties}," + + " audit_info = #{newCatalogMeta.auditInfo}," + + " current_version = #{newCatalogMeta.currentVersion}," + + " last_version = #{newCatalogMeta.lastVersion}," + + " deleted_at = #{newCatalogMeta.deletedAt}" + + " WHERE catalog_id = #{oldCatalogMeta.catalogId}" + + " AND catalog_name = #{oldCatalogMeta.catalogName}" + + " AND metalake_id = #{oldCatalogMeta.metalakeId}" + + " AND type = #{oldCatalogMeta.type}" + + " AND provider = #{oldCatalogMeta.provider}" + + " AND (catalog_comment = #{oldCatalogMeta.catalogComment} " + + " OR (catalog_comment IS NULL and #{oldCatalogMeta.catalogComment} IS NULL))" + + " AND properties = #{oldCatalogMeta.properties}" + + " AND audit_info = #{oldCatalogMeta.auditInfo}" + + " AND current_version = #{oldCatalogMeta.currentVersion}" + + " AND last_version = #{oldCatalogMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteCatalogMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java index faedbcd9642..01cadbb6e8b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.CatalogPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for catalog meta operation SQLs. @@ -38,149 +38,50 @@ public interface CatalogMetaMapper { String TABLE_NAME = "catalog_meta"; - @Select( - "SELECT catalog_id as catalogId, catalog_name as catalogName," - + " metalake_id as metalakeId, type, provider," - + " catalog_comment as catalogComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "listCatalogPOsByMetalakeId") List listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId); - @Select( - "SELECT catalog_id as catalogId FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0") + @SelectProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "selectCatalogIdByMetalakeIdAndName") Long selectCatalogIdByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); - @Select( - "SELECT catalog_id as catalogId, catalog_name as catalogName," - + " metalake_id as metalakeId, type, provider," - + " catalog_comment as catalogComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0") + @SelectProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "selectCatalogMetaByMetalakeIdAndName") CatalogPO selectCatalogMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); - @Select( - "SELECT catalog_id as catalogId, catalog_name as catalogName," - + " metalake_id as metalakeId, type, provider," - + " catalog_comment as catalogComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "selectCatalogMetaById") CatalogPO selectCatalogMetaById(@Param("catalogId") Long catalogId); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(catalog_id, catalog_name, metalake_id," - + " type, provider, catalog_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{catalogMeta.catalogId}," - + " #{catalogMeta.catalogName}," - + " #{catalogMeta.metalakeId}," - + " #{catalogMeta.type}," - + " #{catalogMeta.provider}," - + " #{catalogMeta.catalogComment}," - + " #{catalogMeta.properties}," - + " #{catalogMeta.auditInfo}," - + " #{catalogMeta.currentVersion}," - + " #{catalogMeta.lastVersion}," - + " #{catalogMeta.deletedAt}" - + " )") + @InsertProvider(type = CatalogMetaSQLProviderFactory.class, method = "insertCatalogMeta") void insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(catalog_id, catalog_name, metalake_id," - + " type, provider, catalog_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{catalogMeta.catalogId}," - + " #{catalogMeta.catalogName}," - + " #{catalogMeta.metalakeId}," - + " #{catalogMeta.type}," - + " #{catalogMeta.provider}," - + " #{catalogMeta.catalogComment}," - + " #{catalogMeta.properties}," - + " #{catalogMeta.auditInfo}," - + " #{catalogMeta.currentVersion}," - + " #{catalogMeta.lastVersion}," - + " #{catalogMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " catalog_name = #{catalogMeta.catalogName}," - + " metalake_id = #{catalogMeta.metalakeId}," - + " type = #{catalogMeta.type}," - + " provider = #{catalogMeta.provider}," - + " catalog_comment = #{catalogMeta.catalogComment}," - + " properties = #{catalogMeta.properties}," - + " audit_info = #{catalogMeta.auditInfo}," - + " current_version = #{catalogMeta.currentVersion}," - + " last_version = #{catalogMeta.lastVersion}," - + " deleted_at = #{catalogMeta.deletedAt}") + @InsertProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "insertCatalogMetaOnDuplicateKeyUpdate") void insertCatalogMetaOnDuplicateKeyUpdate(@Param("catalogMeta") CatalogPO catalogPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET catalog_name = #{newCatalogMeta.catalogName}," - + " metalake_id = #{newCatalogMeta.metalakeId}," - + " type = #{newCatalogMeta.type}," - + " provider = #{newCatalogMeta.provider}," - + " catalog_comment = #{newCatalogMeta.catalogComment}," - + " properties = #{newCatalogMeta.properties}," - + " audit_info = #{newCatalogMeta.auditInfo}," - + " current_version = #{newCatalogMeta.currentVersion}," - + " last_version = #{newCatalogMeta.lastVersion}," - + " deleted_at = #{newCatalogMeta.deletedAt}" - + " WHERE catalog_id = #{oldCatalogMeta.catalogId}" - + " AND catalog_name = #{oldCatalogMeta.catalogName}" - + " AND metalake_id = #{oldCatalogMeta.metalakeId}" - + " AND type = #{oldCatalogMeta.type}" - + " AND provider = #{oldCatalogMeta.provider}" - + " AND (catalog_comment = #{oldCatalogMeta.catalogComment} " - + " OR (catalog_comment IS NULL and #{oldCatalogMeta.catalogComment} IS NULL))" - + " AND properties = #{oldCatalogMeta.properties}" - + " AND audit_info = #{oldCatalogMeta.auditInfo}" - + " AND current_version = #{oldCatalogMeta.currentVersion}" - + " AND last_version = #{oldCatalogMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = CatalogMetaSQLProviderFactory.class, method = "updateCatalogMeta") Integer updateCatalogMeta( @Param("newCatalogMeta") CatalogPO newCatalogPO, @Param("oldCatalogMeta") CatalogPO oldCatalogPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "softDeleteCatalogMetasByCatalogId") Integer softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "softDeleteCatalogMetasByMetalakeId") Integer softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "deleteCatalogMetasByLegacyTimeline") Integer deleteCatalogMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java new file mode 100644 index 00000000000..632681c5f0c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.CatalogMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.CatalogPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class CatalogMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new CatalogMetaMySQLProvider(), + JDBCBackendType.H2, new CatalogMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new CatalogMetaPostgreSQLProvider()); + + public static CatalogMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class CatalogMetaMySQLProvider extends CatalogMetaBaseSQLProvider {} + + static class CatalogMetaH2Provider extends CatalogMetaBaseSQLProvider {} + + public static String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().listCatalogPOsByMetalakeId(metalakeId); + } + + public static String selectCatalogIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return getProvider().selectCatalogIdByMetalakeIdAndName(metalakeId, name); + } + + public static String selectCatalogMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("catalogName") String name) { + return getProvider().selectCatalogMetaByMetalakeIdAndName(metalakeId, name); + } + + public static String selectCatalogMetaById(@Param("catalogId") Long catalogId) { + return getProvider().selectCatalogMetaById(catalogId); + } + + public static String insertCatalogMeta(@Param("catalogMeta") CatalogPO catalogPO) { + return getProvider().insertCatalogMeta(catalogPO); + } + + public static String insertCatalogMetaOnDuplicateKeyUpdate( + @Param("catalogMeta") CatalogPO catalogPO) { + return getProvider().insertCatalogMetaOnDuplicateKeyUpdate(catalogPO); + } + + public static String updateCatalogMeta( + @Param("newCatalogMeta") CatalogPO newCatalogPO, + @Param("oldCatalogMeta") CatalogPO oldCatalogPO) { + return getProvider().updateCatalogMeta(newCatalogPO, oldCatalogPO); + } + + public static String softDeleteCatalogMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteCatalogMetasByCatalogId(catalogId); + } + + public static String softDeleteCatalogMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteCatalogMetasByMetalakeId(metalakeId); + } + + public static String deleteCatalogMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteCatalogMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaBaseSQLProvider.java new file mode 100644 index 00000000000..fc79e6d8ead --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaBaseSQLProvider.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper.META_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper.VERSION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.FilesetPO; +import org.apache.ibatis.annotations.Param; + +public class FilesetMetaBaseSQLProvider { + + public String listFilesetPOsBySchemaId(@Param("schemaId") Long schemaId) { + return "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.schema_id = #{schemaId} AND fm.deleted_at = 0 AND vi.deleted_at = 0"; + } + + public String selectFilesetIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return "SELECT fileset_id as filesetId FROM " + + META_TABLE_NAME + + " WHERE schema_id = #{schemaId} AND fileset_name = #{filesetName}" + + " AND deleted_at = 0"; + } + + public String selectFilesetMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.schema_id = #{schemaId} AND fm.fileset_name = #{filesetName}" + + " AND fm.deleted_at = 0 AND vi.deleted_at = 0"; + } + + public String selectFilesetMetaById(@Param("filesetId") Long filesetId) { + return "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.fileset_id = #{filesetId}" + + " AND fm.deleted_at = 0 AND vi.deleted_at = 0"; + } + + public String insertFilesetMeta(@Param("filesetMeta") FilesetPO filesetPO) { + return "INSERT INTO " + + META_TABLE_NAME + + "(fileset_id, fileset_name, metalake_id," + + " catalog_id, schema_id, type, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{filesetMeta.filesetId}," + + " #{filesetMeta.filesetName}," + + " #{filesetMeta.metalakeId}," + + " #{filesetMeta.catalogId}," + + " #{filesetMeta.schemaId}," + + " #{filesetMeta.type}," + + " #{filesetMeta.auditInfo}," + + " #{filesetMeta.currentVersion}," + + " #{filesetMeta.lastVersion}," + + " #{filesetMeta.deletedAt}" + + " )"; + } + + public String insertFilesetMetaOnDuplicateKeyUpdate(@Param("filesetMeta") FilesetPO filesetPO) { + return "INSERT INTO " + + META_TABLE_NAME + + "(fileset_id, fileset_name, metalake_id," + + " catalog_id, schema_id, type, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{filesetMeta.filesetId}," + + " #{filesetMeta.filesetName}," + + " #{filesetMeta.metalakeId}," + + " #{filesetMeta.catalogId}," + + " #{filesetMeta.schemaId}," + + " #{filesetMeta.type}," + + " #{filesetMeta.auditInfo}," + + " #{filesetMeta.currentVersion}," + + " #{filesetMeta.lastVersion}," + + " #{filesetMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " fileset_name = #{filesetMeta.filesetName}," + + " metalake_id = #{filesetMeta.metalakeId}," + + " catalog_id = #{filesetMeta.catalogId}," + + " schema_id = #{filesetMeta.schemaId}," + + " type = #{filesetMeta.type}," + + " audit_info = #{filesetMeta.auditInfo}," + + " current_version = #{filesetMeta.currentVersion}," + + " last_version = #{filesetMeta.lastVersion}," + + " deleted_at = #{filesetMeta.deletedAt}"; + } + + public String updateFilesetMeta( + @Param("newFilesetMeta") FilesetPO newFilesetPO, + @Param("oldFilesetMeta") FilesetPO oldFilesetPO) { + return "UPDATE " + + META_TABLE_NAME + + " SET fileset_name = #{newFilesetMeta.filesetName}," + + " metalake_id = #{newFilesetMeta.metalakeId}," + + " catalog_id = #{newFilesetMeta.catalogId}," + + " schema_id = #{newFilesetMeta.schemaId}," + + " type = #{newFilesetMeta.type}," + + " audit_info = #{newFilesetMeta.auditInfo}," + + " current_version = #{newFilesetMeta.currentVersion}," + + " last_version = #{newFilesetMeta.lastVersion}," + + " deleted_at = #{newFilesetMeta.deletedAt}" + + " WHERE fileset_id = #{oldFilesetMeta.filesetId}" + + " AND fileset_name = #{oldFilesetMeta.filesetName}" + + " AND metalake_id = #{oldFilesetMeta.metalakeId}" + + " AND catalog_id = #{oldFilesetMeta.catalogId}" + + " AND schema_id = #{oldFilesetMeta.schemaId}" + + " AND type = #{oldFilesetMeta.type}" + + " AND audit_info = #{oldFilesetMeta.auditInfo}" + + " AND current_version = #{oldFilesetMeta.currentVersion}" + + " AND last_version = #{oldFilesetMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + public String deleteFilesetMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + META_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java index 61e4d2f95ff..8692f8f890d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaMapper.java @@ -21,13 +21,13 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.FilesetPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Result; import org.apache.ibatis.annotations.Results; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for fileset meta operation SQLs. @@ -42,19 +42,6 @@ public interface FilesetMetaMapper { String VERSION_TABLE_NAME = "fileset_version_info"; - @Select( - "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," - + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," - + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," - + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," - + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," - + " vi.deleted_at as version_deleted_at" - + " FROM " - + META_TABLE_NAME - + " fm INNER JOIN " - + VERSION_TABLE_NAME - + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" - + " WHERE fm.schema_id = #{schemaId} AND fm.deleted_at = 0 AND vi.deleted_at = 0") @Results({ @Result(property = "filesetId", column = "fileset_id"), @Result(property = "filesetName", column = "fileset_name"), @@ -77,30 +64,15 @@ public interface FilesetMetaMapper { @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") }) + @SelectProvider(type = FilesetMetaSQLProviderFactory.class, method = "listFilesetPOsBySchemaId") List listFilesetPOsBySchemaId(@Param("schemaId") Long schemaId); - @Select( - "SELECT fileset_id as filesetId FROM " - + META_TABLE_NAME - + " WHERE schema_id = #{schemaId} AND fileset_name = #{filesetName}" - + " AND deleted_at = 0") + @SelectProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "selectFilesetIdBySchemaIdAndName") Long selectFilesetIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("filesetName") String name); - @Select( - "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," - + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," - + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," - + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," - + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," - + " vi.deleted_at as version_deleted_at" - + " FROM " - + META_TABLE_NAME - + " fm INNER JOIN " - + VERSION_TABLE_NAME - + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" - + " WHERE fm.schema_id = #{schemaId} AND fm.fileset_name = #{filesetName}" - + " AND fm.deleted_at = 0 AND vi.deleted_at = 0") @Results({ @Result(property = "filesetId", column = "fileset_id"), @Result(property = "filesetName", column = "fileset_name"), @@ -123,23 +95,12 @@ Long selectFilesetIdBySchemaIdAndName( @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") }) + @SelectProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "selectFilesetMetaBySchemaIdAndName") FilesetPO selectFilesetMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("filesetName") String name); - @Select( - "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," - + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," - + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," - + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," - + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," - + " vi.deleted_at as version_deleted_at" - + " FROM " - + META_TABLE_NAME - + " fm INNER JOIN " - + VERSION_TABLE_NAME - + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" - + " WHERE fm.fileset_id = #{filesetId}" - + " AND fm.deleted_at = 0 AND vi.deleted_at = 0") @Results({ @Result(property = "filesetId", column = "fileset_id"), @Result(property = "filesetName", column = "fileset_name"), @@ -162,120 +123,45 @@ FilesetPO selectFilesetMetaBySchemaIdAndName( @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") }) + @SelectProvider(type = FilesetMetaSQLProviderFactory.class, method = "selectFilesetMetaById") FilesetPO selectFilesetMetaById(@Param("filesetId") Long filesetId); - @Insert( - "INSERT INTO " - + META_TABLE_NAME - + "(fileset_id, fileset_name, metalake_id," - + " catalog_id, schema_id, type, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{filesetMeta.filesetId}," - + " #{filesetMeta.filesetName}," - + " #{filesetMeta.metalakeId}," - + " #{filesetMeta.catalogId}," - + " #{filesetMeta.schemaId}," - + " #{filesetMeta.type}," - + " #{filesetMeta.auditInfo}," - + " #{filesetMeta.currentVersion}," - + " #{filesetMeta.lastVersion}," - + " #{filesetMeta.deletedAt}" - + " )") + @InsertProvider(type = FilesetMetaSQLProviderFactory.class, method = "insertFilesetMeta") void insertFilesetMeta(@Param("filesetMeta") FilesetPO filesetPO); - @Insert( - "INSERT INTO " - + META_TABLE_NAME - + "(fileset_id, fileset_name, metalake_id," - + " catalog_id, schema_id, type, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{filesetMeta.filesetId}," - + " #{filesetMeta.filesetName}," - + " #{filesetMeta.metalakeId}," - + " #{filesetMeta.catalogId}," - + " #{filesetMeta.schemaId}," - + " #{filesetMeta.type}," - + " #{filesetMeta.auditInfo}," - + " #{filesetMeta.currentVersion}," - + " #{filesetMeta.lastVersion}," - + " #{filesetMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " fileset_name = #{filesetMeta.filesetName}," - + " metalake_id = #{filesetMeta.metalakeId}," - + " catalog_id = #{filesetMeta.catalogId}," - + " schema_id = #{filesetMeta.schemaId}," - + " type = #{filesetMeta.type}," - + " audit_info = #{filesetMeta.auditInfo}," - + " current_version = #{filesetMeta.currentVersion}," - + " last_version = #{filesetMeta.lastVersion}," - + " deleted_at = #{filesetMeta.deletedAt}") + @InsertProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "insertFilesetMetaOnDuplicateKeyUpdate") void insertFilesetMetaOnDuplicateKeyUpdate(@Param("filesetMeta") FilesetPO filesetPO); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET fileset_name = #{newFilesetMeta.filesetName}," - + " metalake_id = #{newFilesetMeta.metalakeId}," - + " catalog_id = #{newFilesetMeta.catalogId}," - + " schema_id = #{newFilesetMeta.schemaId}," - + " type = #{newFilesetMeta.type}," - + " audit_info = #{newFilesetMeta.auditInfo}," - + " current_version = #{newFilesetMeta.currentVersion}," - + " last_version = #{newFilesetMeta.lastVersion}," - + " deleted_at = #{newFilesetMeta.deletedAt}" - + " WHERE fileset_id = #{oldFilesetMeta.filesetId}" - + " AND fileset_name = #{oldFilesetMeta.filesetName}" - + " AND metalake_id = #{oldFilesetMeta.metalakeId}" - + " AND catalog_id = #{oldFilesetMeta.catalogId}" - + " AND schema_id = #{oldFilesetMeta.schemaId}" - + " AND type = #{oldFilesetMeta.type}" - + " AND audit_info = #{oldFilesetMeta.auditInfo}" - + " AND current_version = #{oldFilesetMeta.currentVersion}" - + " AND last_version = #{oldFilesetMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = FilesetMetaSQLProviderFactory.class, method = "updateFilesetMeta") Integer updateFilesetMeta( @Param("newFilesetMeta") FilesetPO newFilesetPO, @Param("oldFilesetMeta") FilesetPO oldFilesetPO); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasByMetalakeId") Integer softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasByCatalogId") Integer softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasBySchemaId") Integer softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId); - @Update( - "UPDATE " - + META_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "softDeleteFilesetMetasByFilesetId") Integer softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId); - @Delete( - "DELETE FROM " - + META_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = FilesetMetaSQLProviderFactory.class, + method = "deleteFilesetMetasByLegacyTimeline") Integer deleteFilesetMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java new file mode 100644 index 00000000000..b41237e2a79 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetMetaSQLProviderFactory.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.FilesetMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.FilesetPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class FilesetMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new FilesetMetaMySQLProvider(), + JDBCBackendType.H2, new FilesetMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new FilesetMetaPostgreSQLProvider()); + + public static FilesetMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class FilesetMetaMySQLProvider extends FilesetMetaBaseSQLProvider {} + + static class FilesetMetaH2Provider extends FilesetMetaBaseSQLProvider {} + + public static String listFilesetPOsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().listFilesetPOsBySchemaId(schemaId); + } + + public static String selectFilesetIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return getProvider().selectFilesetIdBySchemaIdAndName(schemaId, name); + } + + public static String selectFilesetMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("filesetName") String name) { + return getProvider().selectFilesetMetaBySchemaIdAndName(schemaId, name); + } + + public static String selectFilesetMetaById(@Param("filesetId") Long filesetId) { + return getProvider().selectFilesetMetaById(filesetId); + } + + public static String insertFilesetMeta(@Param("filesetMeta") FilesetPO filesetPO) { + return getProvider().insertFilesetMeta(filesetPO); + } + + public static String insertFilesetMetaOnDuplicateKeyUpdate( + @Param("filesetMeta") FilesetPO filesetPO) { + return getProvider().insertFilesetMetaOnDuplicateKeyUpdate(filesetPO); + } + + public static String updateFilesetMeta( + @Param("newFilesetMeta") FilesetPO newFilesetPO, + @Param("oldFilesetMeta") FilesetPO oldFilesetPO) { + return getProvider().updateFilesetMeta(newFilesetPO, oldFilesetPO); + } + + public static String softDeleteFilesetMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteFilesetMetasByMetalakeId(metalakeId); + } + + public static String softDeleteFilesetMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteFilesetMetasByCatalogId(catalogId); + } + + public static String softDeleteFilesetMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteFilesetMetasBySchemaId(schemaId); + } + + public String softDeleteFilesetMetasByFilesetId(@Param("filesetId") Long filesetId) { + return getProvider().softDeleteFilesetMetasByFilesetId(filesetId); + } + + public String deleteFilesetMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteFilesetMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionBaseSQLProvider.java new file mode 100644 index 00000000000..f6ab85b38be --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionBaseSQLProvider.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper.VERSION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.FilesetVersionPO; +import org.apache.ibatis.annotations.Param; + +public class FilesetVersionBaseSQLProvider { + public String insertFilesetVersion(@Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return "INSERT INTO " + + VERSION_TABLE_NAME + + "(metalake_id, catalog_id, schema_id, fileset_id," + + " version, fileset_comment, properties, storage_location," + + " deleted_at)" + + " VALUES(" + + " #{filesetVersion.metalakeId}," + + " #{filesetVersion.catalogId}," + + " #{filesetVersion.schemaId}," + + " #{filesetVersion.filesetId}," + + " #{filesetVersion.version}," + + " #{filesetVersion.filesetComment}," + + " #{filesetVersion.properties}," + + " #{filesetVersion.storageLocation}," + + " #{filesetVersion.deletedAt}" + + " )"; + } + + public String insertFilesetVersionOnDuplicateKeyUpdate( + @Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return "INSERT INTO " + + VERSION_TABLE_NAME + + "(metalake_id, catalog_id, schema_id, fileset_id," + + " version, fileset_comment, properties, storage_location," + + " deleted_at)" + + " VALUES(" + + " #{filesetVersion.metalakeId}," + + " #{filesetVersion.catalogId}," + + " #{filesetVersion.schemaId}," + + " #{filesetVersion.filesetId}," + + " #{filesetVersion.version}," + + " #{filesetVersion.filesetComment}," + + " #{filesetVersion.properties}," + + " #{filesetVersion.storageLocation}," + + " #{filesetVersion.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " metalake_id = #{filesetVersion.metalakeId}," + + " catalog_id = #{filesetVersion.catalogId}," + + " schema_id = #{filesetVersion.schemaId}," + + " fileset_id = #{filesetVersion.filesetId}," + + " version = #{filesetVersion.version}," + + " fileset_comment = #{filesetVersion.filesetComment}," + + " properties = #{filesetVersion.properties}," + + " storage_location = #{filesetVersion.storageLocation}," + + " deleted_at = #{filesetVersion.deletedAt}"; + } + + public String softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + public String deleteFilesetVersionsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + VERSION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } + + public String selectFilesetVersionsByRetentionCount( + @Param("versionRetentionCount") Long versionRetentionCount) { + return "SELECT fileset_id as filesetId," + + " Max(version) as version" + + " FROM " + + VERSION_TABLE_NAME + + " WHERE version > #{versionRetentionCount} AND deleted_at = 0" + + " GROUP BY fileset_id"; + } + + public String softDeleteFilesetVersionsByRetentionLine( + @Param("filesetId") Long filesetId, + @Param("versionRetentionLine") long versionRetentionLine, + @Param("limit") int limit) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java index cd4bfb9b613..09eca4b90e0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionMapper.java @@ -22,11 +22,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.FilesetMaxVersionPO; import org.apache.gravitino.storage.relational.po.FilesetVersionPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for fileset version info operation SQLs. @@ -39,110 +39,50 @@ public interface FilesetVersionMapper { String VERSION_TABLE_NAME = "fileset_version_info"; - @Insert( - "INSERT INTO " - + VERSION_TABLE_NAME - + "(metalake_id, catalog_id, schema_id, fileset_id," - + " version, fileset_comment, properties, storage_location," - + " deleted_at)" - + " VALUES(" - + " #{filesetVersion.metalakeId}," - + " #{filesetVersion.catalogId}," - + " #{filesetVersion.schemaId}," - + " #{filesetVersion.filesetId}," - + " #{filesetVersion.version}," - + " #{filesetVersion.filesetComment}," - + " #{filesetVersion.properties}," - + " #{filesetVersion.storageLocation}," - + " #{filesetVersion.deletedAt}" - + " )") + @InsertProvider(type = FilesetVersionSQLProviderFactory.class, method = "insertFilesetVersion") void insertFilesetVersion(@Param("filesetVersion") FilesetVersionPO filesetVersionPO); - @Insert( - "INSERT INTO " - + VERSION_TABLE_NAME - + "(metalake_id, catalog_id, schema_id, fileset_id," - + " version, fileset_comment, properties, storage_location," - + " deleted_at)" - + " VALUES(" - + " #{filesetVersion.metalakeId}," - + " #{filesetVersion.catalogId}," - + " #{filesetVersion.schemaId}," - + " #{filesetVersion.filesetId}," - + " #{filesetVersion.version}," - + " #{filesetVersion.filesetComment}," - + " #{filesetVersion.properties}," - + " #{filesetVersion.storageLocation}," - + " #{filesetVersion.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " metalake_id = #{filesetVersion.metalakeId}," - + " catalog_id = #{filesetVersion.catalogId}," - + " schema_id = #{filesetVersion.schemaId}," - + " fileset_id = #{filesetVersion.filesetId}," - + " version = #{filesetVersion.version}," - + " fileset_comment = #{filesetVersion.filesetComment}," - + " properties = #{filesetVersion.properties}," - + " storage_location = #{filesetVersion.storageLocation}," - + " deleted_at = #{filesetVersion.deletedAt}") + @InsertProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "insertFilesetVersionOnDuplicateKeyUpdate") void insertFilesetVersionOnDuplicateKeyUpdate( @Param("filesetVersion") FilesetVersionPO filesetVersionPO); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByMetalakeId") Integer softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByCatalogId") Integer softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsBySchemaId") Integer softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE fileset_id = #{filesetId} AND deleted_at = 0") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByFilesetId") Integer softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId); - @Delete( - "DELETE FROM " - + VERSION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "deleteFilesetVersionsByLegacyTimeline") Integer deleteFilesetVersionsByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); - @Select( - "SELECT fileset_id as filesetId," - + " Max(version) as version" - + " FROM " - + VERSION_TABLE_NAME - + " WHERE version > #{versionRetentionCount} AND deleted_at = 0" - + " GROUP BY fileset_id") + @SelectProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "selectFilesetVersionsByRetentionCount") List selectFilesetVersionsByRetentionCount( @Param("versionRetentionCount") Long versionRetentionCount); - @Update( - "UPDATE " - + VERSION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}") + @UpdateProvider( + type = FilesetVersionSQLProviderFactory.class, + method = "softDeleteFilesetVersionsByRetentionLine") Integer softDeleteFilesetVersionsByRetentionLine( @Param("filesetId") Long filesetId, @Param("versionRetentionLine") long versionRetentionLine, diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java new file mode 100644 index 00000000000..4029c2cb055 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/FilesetVersionSQLProviderFactory.java @@ -0,0 +1,95 @@ +package org.apache.gravitino.storage.relational.mapper; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.FilesetVersionPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.FilesetVersionPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class FilesetVersionSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new FilesetVersionMySQLProvider(), + JDBCBackendType.H2, new FilesetVersionH2Provider(), + JDBCBackendType.POSTGRESQL, new FilesetVersionPostgreSQLProvider()); + + public static FilesetVersionBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class FilesetVersionMySQLProvider extends FilesetVersionBaseSQLProvider {} + + static class FilesetVersionH2Provider extends FilesetVersionBaseSQLProvider {} + + public static String insertFilesetVersion( + @Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return getProvider().insertFilesetVersion(filesetVersionPO); + } + + public static String insertFilesetVersionOnDuplicateKeyUpdate( + @Param("filesetVersion") FilesetVersionPO filesetVersionPO) { + return getProvider().insertFilesetVersionOnDuplicateKeyUpdate(filesetVersionPO); + } + + public static String softDeleteFilesetVersionsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteFilesetVersionsByMetalakeId(metalakeId); + } + + public static String softDeleteFilesetVersionsByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteFilesetVersionsByCatalogId(catalogId); + } + + public static String softDeleteFilesetVersionsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteFilesetVersionsBySchemaId(schemaId); + } + + public static String softDeleteFilesetVersionsByFilesetId(@Param("filesetId") Long filesetId) { + return getProvider().softDeleteFilesetVersionsByFilesetId(filesetId); + } + + public static String deleteFilesetVersionsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteFilesetVersionsByLegacyTimeline(legacyTimeline, limit); + } + + public static String selectFilesetVersionsByRetentionCount( + @Param("versionRetentionCount") Long versionRetentionCount) { + return getProvider().selectFilesetVersionsByRetentionCount(versionRetentionCount); + } + + public static String softDeleteFilesetVersionsByRetentionLine( + @Param("filesetId") Long filesetId, + @Param("versionRetentionLine") long versionRetentionLine, + @Param("limit") int limit) { + return getProvider() + .softDeleteFilesetVersionsByRetentionLine(filesetId, versionRetentionLine, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaBaseSQLProvider.java new file mode 100644 index 00000000000..686bc42c565 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaBaseSQLProvider.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.GroupMetaMapper.GROUP_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.GROUP_ROLE_RELATION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.GroupPO; +import org.apache.ibatis.annotations.Param; + +public class GroupMetaBaseSQLProvider { + + public String selectGroupIdBySchemaIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return "SELECT group_id as groupId FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" + + " AND deleted_at = 0"; + } + + public String selectGroupMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return "SELECT group_id as groupId, group_name as groupName," + + " metalake_id as metalakeId," + + " audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" + + " AND deleted_at = 0"; + } + + public String insertGroupMeta(@Param("groupMeta") GroupPO groupPO) { + return "INSERT INTO " + + GROUP_TABLE_NAME + + "(group_id, group_name," + + " metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )"; + } + + public String insertGroupMetaOnDuplicateKeyUpdate(@Param("groupMeta") GroupPO groupPO) { + return "INSERT INTO " + + GROUP_TABLE_NAME + + "(group_id, group_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " group_name = #{groupMeta.groupName}," + + " metalake_id = #{groupMeta.metalakeId}," + + " audit_info = #{groupMeta.auditInfo}," + + " current_version = #{groupMeta.currentVersion}," + + " last_version = #{groupMeta.lastVersion}," + + " deleted_at = #{groupMeta.deletedAt}"; + } + + public String softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + public String softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String updateGroupMeta( + @Param("newGroupMeta") GroupPO newGroupPO, @Param("oldGroupMeta") GroupPO oldGroupPO) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET group_name = #{newGroupMeta.groupName}," + + " metalake_id = #{newGroupMeta.metalakeId}," + + " audit_info = #{newGroupMeta.auditInfo}," + + " current_version = #{newGroupMeta.currentVersion}," + + " last_version = #{newGroupMeta.lastVersion}," + + " deleted_at = #{newGroupMeta.deletedAt}" + + " WHERE group_id = #{oldGroupMeta.groupId}" + + " AND group_name = #{oldGroupMeta.groupName}" + + " AND metalake_id = #{oldGroupMeta.metalakeId}" + + " AND audit_info = #{oldGroupMeta.auditInfo}" + + " AND current_version = #{oldGroupMeta.currentVersion}" + + " AND last_version = #{oldGroupMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String listGroupsByRoleId(@Param("roleId") Long roleId) { + return "SELECT gr.group_id as groupId, gr.group_name as groupName," + + " gr.metalake_id as metalakeId," + + " gr.audit_info as auditInfo, gr.current_version as currentVersion," + + " gr.last_version as lastVersion, gr.deleted_at as deletedAt" + + " FROM " + + GROUP_TABLE_NAME + + " gr JOIN " + + GROUP_ROLE_RELATION_TABLE_NAME + + " re ON gr.group_id = re.group_id" + + " WHERE re.role_id = #{roleId}" + + " AND gr.deleted_at = 0 AND re.deleted_at = 0"; + } + + public String deleteGroupMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + GROUP_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java index 6250c2a9a6c..5743095dd72 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.GroupPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -39,121 +39,44 @@ public interface GroupMetaMapper { String GROUP_TABLE_NAME = "group_meta"; String GROUP_ROLE_RELATION_TABLE_NAME = "group_role_rel"; - @Select( - "SELECT group_id as groupId FROM " - + GROUP_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" - + " AND deleted_at = 0") + @SelectProvider( + type = GroupMetaSQLProviderFactory.class, + method = "selectGroupIdBySchemaIdAndName") Long selectGroupIdBySchemaIdAndName( @Param("metalakeId") Long metalakeId, @Param("groupName") String name); - @Select( - "SELECT group_id as groupId, group_name as groupName," - + " metalake_id as metalakeId," - + " audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + GROUP_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND group_name = #{groupName}" - + " AND deleted_at = 0") + @SelectProvider( + type = GroupMetaSQLProviderFactory.class, + method = "selectGroupMetaByMetalakeIdAndName") GroupPO selectGroupMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("groupName") String name); - @Insert( - "INSERT INTO " - + GROUP_TABLE_NAME - + "(group_id, group_name," - + " metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{groupMeta.groupId}," - + " #{groupMeta.groupName}," - + " #{groupMeta.metalakeId}," - + " #{groupMeta.auditInfo}," - + " #{groupMeta.currentVersion}," - + " #{groupMeta.lastVersion}," - + " #{groupMeta.deletedAt}" - + " )") + @InsertProvider(type = GroupMetaSQLProviderFactory.class, method = "insertGroupMeta") void insertGroupMeta(@Param("groupMeta") GroupPO groupPO); - @Insert( - "INSERT INTO " - + GROUP_TABLE_NAME - + "(group_id, group_name," - + "metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{groupMeta.groupId}," - + " #{groupMeta.groupName}," - + " #{groupMeta.metalakeId}," - + " #{groupMeta.auditInfo}," - + " #{groupMeta.currentVersion}," - + " #{groupMeta.lastVersion}," - + " #{groupMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " group_name = #{groupMeta.groupName}," - + " metalake_id = #{groupMeta.metalakeId}," - + " audit_info = #{groupMeta.auditInfo}," - + " current_version = #{groupMeta.currentVersion}," - + " last_version = #{groupMeta.lastVersion}," - + " deleted_at = #{groupMeta.deletedAt}") + @InsertProvider( + type = GroupMetaSQLProviderFactory.class, + method = "insertGroupMetaOnDuplicateKeyUpdate") void insertGroupMetaOnDuplicateKeyUpdate(@Param("groupMeta") GroupPO groupPO); - @Update( - "UPDATE " - + GROUP_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE group_id = #{groupId} AND deleted_at = 0") + @UpdateProvider(type = GroupMetaSQLProviderFactory.class, method = "softDeleteGroupMetaByGroupId") void softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId); - @Update( - "UPDATE " - + GROUP_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = GroupMetaSQLProviderFactory.class, + method = "softDeleteGroupMetasByMetalakeId") void softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + GROUP_TABLE_NAME - + " SET group_name = #{newGroupMeta.groupName}," - + " metalake_id = #{newGroupMeta.metalakeId}," - + " audit_info = #{newGroupMeta.auditInfo}," - + " current_version = #{newGroupMeta.currentVersion}," - + " last_version = #{newGroupMeta.lastVersion}," - + " deleted_at = #{newGroupMeta.deletedAt}" - + " WHERE group_id = #{oldGroupMeta.groupId}" - + " AND group_name = #{oldGroupMeta.groupName}" - + " AND metalake_id = #{oldGroupMeta.metalakeId}" - + " AND audit_info = #{oldGroupMeta.auditInfo}" - + " AND current_version = #{oldGroupMeta.currentVersion}" - + " AND last_version = #{oldGroupMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = GroupMetaSQLProviderFactory.class, method = "updateGroupMeta") Integer updateGroupMeta( @Param("newGroupMeta") GroupPO newGroupPO, @Param("oldGroupMeta") GroupPO oldGroupPO); - @Select( - "SELECT gr.group_id as groupId, gr.group_name as groupName," - + " gr.metalake_id as metalakeId," - + " gr.audit_info as auditInfo, gr.current_version as currentVersion," - + " gr.last_version as lastVersion, gr.deleted_at as deletedAt" - + " FROM " - + GROUP_TABLE_NAME - + " gr JOIN " - + GROUP_ROLE_RELATION_TABLE_NAME - + " re ON gr.group_id = re.group_id" - + " WHERE re.role_id = #{roleId}" - + " AND gr.deleted_at = 0 AND re.deleted_at = 0") + @SelectProvider(type = GroupMetaSQLProviderFactory.class, method = "listGroupsByRoleId") List listGroupsByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + GROUP_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = GroupMetaSQLProviderFactory.class, + method = "deleteGroupMetasByLegacyTimeline") Integer deleteGroupMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java new file mode 100644 index 00000000000..2769bf1d959 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupMetaSQLProviderFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.GroupMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.GroupPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class GroupMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new GroupMetaMySQLProvider(), + JDBCBackendType.H2, new GroupMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new GroupMetaPostgreSQLProvider()); + + public static GroupMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class GroupMetaMySQLProvider extends GroupMetaBaseSQLProvider {} + + static class GroupMetaH2Provider extends GroupMetaBaseSQLProvider {} + + public static String selectGroupIdBySchemaIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return getProvider().selectGroupIdBySchemaIdAndName(metalakeId, name); + } + + public static String selectGroupMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("groupName") String name) { + return getProvider().selectGroupMetaByMetalakeIdAndName(metalakeId, name); + } + + public static String insertGroupMeta(@Param("groupMeta") GroupPO groupPO) { + return getProvider().insertGroupMeta(groupPO); + } + + public static String insertGroupMetaOnDuplicateKeyUpdate(@Param("groupMeta") GroupPO groupPO) { + return getProvider().insertGroupMetaOnDuplicateKeyUpdate(groupPO); + } + + public static String softDeleteGroupMetaByGroupId(@Param("groupId") Long groupId) { + return getProvider().softDeleteGroupMetaByGroupId(groupId); + } + + public static String softDeleteGroupMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteGroupMetasByMetalakeId(metalakeId); + } + + public static String updateGroupMeta( + @Param("newGroupMeta") GroupPO newGroupPO, @Param("oldGroupMeta") GroupPO oldGroupPO) { + return getProvider().updateGroupMeta(newGroupPO, oldGroupPO); + } + + public static String listGroupsByRoleId(@Param("roleId") Long roleId) { + return getProvider().listGroupsByRoleId(roleId); + } + + public static String deleteGroupMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteGroupMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelBaseSQLProvider.java new file mode 100644 index 00000000000..ea883681570 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelBaseSQLProvider.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; +import org.apache.ibatis.annotations.Param; + +public class GroupRoleRelBaseSQLProvider { + public String batchInsertGroupRoleRel( + @Param("groupRoleRels") List groupRoleRelPOS) { + return ""; + } + + public String batchInsertGroupRoleRelOnDuplicateKeyUpdate( + @Param("groupRoleRels") List groupRoleRelPOS) { + return ""; + } + + public String softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + public String softDeleteGroupRoleRelByGroupAndRoles( + @Param("groupId") Long groupId, @Param("roleIds") List roleIds) { + return ""; + } + + public String softDeleteGroupRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE group_id IN (SELECT group_id FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + public String softDeleteGroupRoleRelByRoleId(Long roleId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String deleteGroupRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + GROUP_ROLE_RELATION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java index c428341d7a4..a40ee8daae9 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelMapper.java @@ -21,10 +21,9 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -38,101 +37,39 @@ public interface GroupRoleRelMapper { String GROUP_TABLE_NAME = "group_meta"; String GROUP_ROLE_RELATION_TABLE_NAME = "group_role_rel"; - @Insert({ - "" - }) + @InsertProvider(type = GroupRoleRelSQLProviderFactory.class, method = "batchInsertGroupRoleRel") void batchInsertGroupRoleRel(@Param("groupRoleRels") List groupRoleRelPOS); - @Insert({ - "" - }) + @InsertProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "batchInsertGroupRoleRelOnDuplicateKeyUpdate") void batchInsertGroupRoleRelOnDuplicateKeyUpdate( @Param("groupRoleRels") List groupRoleRelPOS); - @Update( - "UPDATE " - + GROUP_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE group_id = #{groupId} AND deleted_at = 0") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByGroupId") void softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId); - @Update({ - "" - }) + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByGroupAndRoles") void softDeleteGroupRoleRelByGroupAndRoles( @Param("groupId") Long groupId, @Param("roleIds") List roleIds); - @Update( - "UPDATE " - + GROUP_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE group_id IN (SELECT group_id FROM " - + GROUP_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" - + " AND deleted_at = 0") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByMetalakeId") void softDeleteGroupRoleRelByMetalakeId(Long metalakeId); - @Update( - "UPDATE " - + GROUP_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "softDeleteGroupRoleRelByRoleId") void softDeleteGroupRoleRelByRoleId(Long roleId); - @Delete( - "DELETE FROM " - + GROUP_ROLE_RELATION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @UpdateProvider( + type = GroupRoleRelSQLProviderFactory.class, + method = "deleteGroupRoleRelMetasByLegacyTimeline") Integer deleteGroupRoleRelMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java new file mode 100644 index 00000000000..6d2ff176bd1 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/GroupRoleRelSQLProviderFactory.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.GroupRoleRelPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class GroupRoleRelSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new GroupRoleRelMySQLProvider(), + JDBCBackendType.H2, new GroupRoleRelH2Provider(), + JDBCBackendType.POSTGRESQL, new GroupRoleRelPostgreSQLProvider()); + + public static GroupRoleRelBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class GroupRoleRelMySQLProvider extends GroupRoleRelBaseSQLProvider {} + + static class GroupRoleRelH2Provider extends GroupRoleRelBaseSQLProvider {} + + public static String batchInsertGroupRoleRel( + @Param("groupRoleRels") List groupRoleRelPOS) { + return getProvider().batchInsertGroupRoleRel(groupRoleRelPOS); + } + + public static String batchInsertGroupRoleRelOnDuplicateKeyUpdate( + @Param("groupRoleRels") List groupRoleRelPOS) { + return getProvider().batchInsertGroupRoleRelOnDuplicateKeyUpdate(groupRoleRelPOS); + } + + public static String softDeleteGroupRoleRelByGroupId(@Param("groupId") Long groupId) { + return getProvider().softDeleteGroupRoleRelByGroupId(groupId); + } + + public static String softDeleteGroupRoleRelByGroupAndRoles( + @Param("groupId") Long groupId, @Param("roleIds") List roleIds) { + return getProvider().softDeleteGroupRoleRelByGroupAndRoles(groupId, roleIds); + } + + public static String softDeleteGroupRoleRelByMetalakeId(Long metalakeId) { + return getProvider().softDeleteGroupRoleRelByMetalakeId(metalakeId); + } + + public static String softDeleteGroupRoleRelByRoleId(Long roleId) { + return getProvider().softDeleteGroupRoleRelByRoleId(roleId); + } + + public static String deleteGroupRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteGroupRoleRelMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaBaseSQLProvider.java new file mode 100644 index 00000000000..a782ddb8f0f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaBaseSQLProvider.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.ibatis.annotations.Param; + +public class MetalakeMetaBaseSQLProvider { + + public String listMetalakePOs() { + return "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE deleted_at = 0"; + } + + public String selectMetalakeMetaByName(@Param("metalakeName") String metalakeName) { + return "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_name = #{metalakeName} and deleted_at = 0"; + } + + public String selectMetalakeMetaById(@Param("metalakeId") Long metalakeId) { + return "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalakeId} and deleted_at = 0"; + } + + public String selectMetalakeIdMetaByName(@Param("metalakeName") String metalakeName) { + return "SELECT metalake_id as metalakeId" + + " FROM " + + TABLE_NAME + + " WHERE metalake_name = #{metalakeName} and deleted_at = 0"; + } + + public String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) { + return "INSERT INTO " + + TABLE_NAME + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{metalakeMeta.metalakeId}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + + " )"; + } + + public String insertMetalakeMetaOnDuplicateKeyUpdate( + @Param("metalakeMeta") MetalakePO metalakePO) { + return "INSERT INTO " + + TABLE_NAME + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{metalakeMeta.metalakeId}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}," + + " current_version = #{metalakeMeta.currentVersion}," + + " last_version = #{metalakeMeta.lastVersion}," + + " deleted_at = #{metalakeMeta.deletedAt}"; + } + + public String updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO) { + return "UPDATE " + + TABLE_NAME + + " SET metalake_name = #{newMetalakeMeta.metalakeName}," + + " metalake_comment = #{newMetalakeMeta.metalakeComment}," + + " properties = #{newMetalakeMeta.properties}," + + " audit_info = #{newMetalakeMeta.auditInfo}," + + " schema_version = #{newMetalakeMeta.schemaVersion}," + + " current_version = #{newMetalakeMeta.currentVersion}," + + " last_version = #{newMetalakeMeta.lastVersion}" + + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" + + " AND metalake_name = #{oldMetalakeMeta.metalakeName}" + + " AND (metalake_comment = #{oldMetalakeMeta.metalakeComment} " + + " OR (metalake_comment IS NULL and #{oldMetalakeMeta.metalakeComment} IS NULL))" + + " AND properties = #{oldMetalakeMeta.properties}" + + " AND audit_info = #{oldMetalakeMeta.auditInfo}" + + " AND schema_version = #{oldMetalakeMeta.schemaVersion}" + + " AND current_version = #{oldMetalakeMeta.currentVersion}" + + " AND last_version = #{oldMetalakeMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteMetalakeMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index 89f8d13ceb1..d5dc809bfe2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.MetalakePO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for metalake meta operation SQLs. @@ -38,126 +38,41 @@ public interface MetalakeMetaMapper { String TABLE_NAME = "metalake_meta"; - @Select( - "SELECT metalake_id as metalakeId, metalake_name as metalakeName," - + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE deleted_at = 0") + @SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "listMetalakePOs") List listMetalakePOs(); - @Select( - "SELECT metalake_id as metalakeId, metalake_name as metalakeName," - + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_name = #{metalakeName} and deleted_at = 0") + @SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeMetaByName") MetalakePO selectMetalakeMetaByName(@Param("metalakeName") String name); - @Select( - "SELECT metalake_id as metalakeId, metalake_name as metalakeName," - + " metalake_comment as metalakeComment, properties," - + " audit_info as auditInfo, schema_version as schemaVersion," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE metalake_id = #{metalakeId} and deleted_at = 0") + @SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeMetaById") MetalakePO selectMetalakeMetaById(@Param("metalakeId") Long metalakeId); - @Select( - "SELECT metalake_id FROM " - + TABLE_NAME - + " WHERE metalake_name = #{metalakeName} and deleted_at = 0") + @SelectProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "selectMetalakeIdMetaByName") Long selectMetalakeIdMetaByName(@Param("metalakeName") String name); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," - + " schema_version, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{metalakeMeta.metalakeId}," - + " #{metalakeMeta.metalakeName}," - + " #{metalakeMeta.metalakeComment}," - + " #{metalakeMeta.properties}," - + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}," - + " #{metalakeMeta.currentVersion}," - + " #{metalakeMeta.lastVersion}," - + " #{metalakeMeta.deletedAt}" - + " )") + @InsertProvider(type = MetalakeMetaSQLProviderFactory.class, method = "insertMetalakeMeta") void insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," - + " schema_version, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{metalakeMeta.metalakeId}," - + " #{metalakeMeta.metalakeName}," - + " #{metalakeMeta.metalakeComment}," - + " #{metalakeMeta.properties}," - + " #{metalakeMeta.auditInfo}," - + " #{metalakeMeta.schemaVersion}," - + " #{metalakeMeta.currentVersion}," - + " #{metalakeMeta.lastVersion}," - + " #{metalakeMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " metalake_name = #{metalakeMeta.metalakeName}," - + " metalake_comment = #{metalakeMeta.metalakeComment}," - + " properties = #{metalakeMeta.properties}," - + " audit_info = #{metalakeMeta.auditInfo}," - + " schema_version = #{metalakeMeta.schemaVersion}," - + " current_version = #{metalakeMeta.currentVersion}," - + " last_version = #{metalakeMeta.lastVersion}," - + " deleted_at = #{metalakeMeta.deletedAt}") + @InsertProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "insertMetalakeMetaOnDuplicateKeyUpdate") void insertMetalakeMetaOnDuplicateKeyUpdate(@Param("metalakeMeta") MetalakePO metalakePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET metalake_name = #{newMetalakeMeta.metalakeName}," - + " metalake_comment = #{newMetalakeMeta.metalakeComment}," - + " properties = #{newMetalakeMeta.properties}," - + " audit_info = #{newMetalakeMeta.auditInfo}," - + " schema_version = #{newMetalakeMeta.schemaVersion}," - + " current_version = #{newMetalakeMeta.currentVersion}," - + " last_version = #{newMetalakeMeta.lastVersion}" - + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" - + " AND metalake_name = #{oldMetalakeMeta.metalakeName}" - + " AND (metalake_comment = #{oldMetalakeMeta.metalakeComment} " - + " OR (metalake_comment IS NULL and #{oldMetalakeMeta.metalakeComment} IS NULL))" - + " AND properties = #{oldMetalakeMeta.properties}" - + " AND audit_info = #{oldMetalakeMeta.auditInfo}" - + " AND schema_version = #{oldMetalakeMeta.schemaVersion}" - + " AND current_version = #{oldMetalakeMeta.currentVersion}" - + " AND last_version = #{oldMetalakeMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = MetalakeMetaSQLProviderFactory.class, method = "updateMetalakeMeta") Integer updateMetalakeMeta( @Param("newMetalakeMeta") MetalakePO newMetalakePO, @Param("oldMetalakeMeta") MetalakePO oldMetalakePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "softDeleteMetalakeMetaByMetalakeId") Integer softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = MetalakeMetaSQLProviderFactory.class, + method = "deleteMetalakeMetasByLegacyTimeline") Integer deleteMetalakeMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java new file mode 100644 index 00000000000..67589459631 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/MetalakeMetaSQLProviderFactory.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.MetalakeMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +/** SQL Provider for Metalake Meta operations. */ +public class MetalakeMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new MetalakeMetaMySQLProvider(), + JDBCBackendType.H2, new MetalakeMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new MetalakeMetaPostgreSQLProvider()); + + public static MetalakeMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class MetalakeMetaMySQLProvider extends MetalakeMetaBaseSQLProvider {} + + static class MetalakeMetaH2Provider extends MetalakeMetaBaseSQLProvider {} + + public String listMetalakePOs() { + return getProvider().listMetalakePOs(); + } + + public static String selectMetalakeMetaByName(@Param("metalakeName") String metalakeName) { + return getProvider().selectMetalakeMetaByName(metalakeName); + } + + public static String selectMetalakeMetaById(@Param("metalakeId") Long metalakeId) { + return getProvider().selectMetalakeMetaById(metalakeId); + } + + public static String selectMetalakeIdMetaByName(@Param("metalakeName") String metalakeName) { + return getProvider().selectMetalakeIdMetaByName(metalakeName); + } + + public static String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) { + return getProvider().insertMetalakeMeta(metalakePO); + } + + public static String insertMetalakeMetaOnDuplicateKeyUpdate( + @Param("metalakeMeta") MetalakePO metalakePO) { + return getProvider().insertMetalakeMetaOnDuplicateKeyUpdate(metalakePO); + } + + public static String updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO) { + return getProvider().updateMetalakeMeta(newMetalakePO, oldMetalakePO); + } + + public static String softDeleteMetalakeMetaByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteMetalakeMetaByMetalakeId(metalakeId); + } + + public static String deleteMetalakeMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteMetalakeMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaBaseSQLProvider.java new file mode 100644 index 00000000000..0d46034a5db --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaBaseSQLProvider.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper.OWNER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.OwnerRelPO; +import org.apache.ibatis.annotations.Param; + +public class OwnerMetaBaseSQLProvider { + public String selectUserOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT ut.user_id as userId," + + " ut.user_name as userName," + + " ut.metalake_id as metalakeId," + + " ut.audit_info as auditInfo," + + " ut.current_version as currentVersion," + + " ut.last_version as lastVersion," + + " ut.deleted_at as deletedAt" + + " FROM " + + OWNER_TABLE_NAME + + " ot JOIN " + + UserMetaMapper.USER_TABLE_NAME + + " ut ON ut.user_id = ot.owner_id" + + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" + + " ot.metadata_object_type = #{metadataObjectType} AND" + + " ot.owner_type = 'USER' AND" + + " ot.deleted_at = 0 AND ut.deleted_at = 0"; + } + + public String selectGroupOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT gt.group_id as groupId," + + " gt.group_name as groupName," + + " gt.metalake_id as metalakeId," + + " gt.audit_info as auditInfo," + + " gt.current_version as currentVersion," + + " gt.last_version as lastVersion," + + " gt.deleted_at as deletedAt" + + " FROM " + + OWNER_TABLE_NAME + + " ot JOIN " + + GroupMetaMapper.GROUP_TABLE_NAME + + " gt ON gt.group_id = ot.owner_id" + + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" + + " ot.metadata_object_type = #{metadataObjectType} AND" + + " ot.owner_type = 'GROUP' AND" + + " ot.deleted_at = 0 AND gt.deleted_at = 0"; + } + + public String insertOwnerRel(@Param("ownerRelPO") OwnerRelPO ownerRelPO) { + return "INSERT INTO " + + OWNER_TABLE_NAME + + "(metalake_id, metadata_object_id, metadata_object_type, owner_id, owner_type," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES (" + + " #{ownerRelPO.metalakeId}," + + " #{ownerRelPO.metadataObjectId}," + + " #{ownerRelPO.metadataObjectType}," + + " #{ownerRelPO.ownerId}," + + " #{ownerRelPO.ownerType}," + + " #{ownerRelPO.auditInfo}," + + " #{ownerRelPO.currentVersion}," + + " #{ownerRelPO.lastVersion}," + + " #{ownerRelPO.deletedAt}" + + ")"; + } + + public String softDeleteOwnerRelByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metadata_object_id = #{metadataObjectId} AND metadata_object_type = #{metadataObjectType} AND deleted_at = 0"; + } + + public String softDeleteOwnerRelByOwnerIdAndType( + @Param("ownerId") Long ownerId, @Param("ownerType") String ownerType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE owner_id = #{ownerId} AND owner_type = #{ownerType} AND deleted_at = 0"; + } + + public String softDeleteOwnerRelByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at =0"; + } + + public String softDeleteOwnerRelByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE EXISTS (" + + " SELECT ct.catalog_id FROM " + + CatalogMetaMapper.TABLE_NAME + + " ct WHERE ct.catalog_id = #{catalogId} AND ct.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'" + + " UNION " + + " SELECT st.catalog_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.catalog_id = #{catalogId} AND st.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.catalog_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.catalog_id = #{catalogId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.catalog_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.catalog_id = #{catalogId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.catalog_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.catalog_id = #{catalogId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND" + + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } + + public String sotDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE EXISTS (" + + " SELECT st.schema_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.schema_id = #{schemaId} AND st.deleted_at = 0 AND ot.deleted_at = 0 " + + "AND st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.schema_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.schema_id = #{schemaId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.schema_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.schema_id = #{schemaId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.schema_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.schema_id = #{schemaId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } + + public String deleteOwnerMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + OWNER_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java index 11c288324bf..455d78fb598 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaMapper.java @@ -21,11 +21,10 @@ import org.apache.gravitino.storage.relational.po.GroupPO; import org.apache.gravitino.storage.relational.po.OwnerRelPO; import org.apache.gravitino.storage.relational.po.UserPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for owner meta operation SQLs. @@ -39,158 +38,52 @@ public interface OwnerMetaMapper { String OWNER_TABLE_NAME = "owner_meta"; - @Select( - "SELECT ut.user_id as userId," - + " ut.user_name as userName," - + " ut.metalake_id as metalakeId," - + " ut.audit_info as auditInfo," - + " ut.current_version as currentVersion," - + " ut.last_version as lastVersion," - + " ut.deleted_at as deletedAt" - + " FROM " - + OWNER_TABLE_NAME - + " ot JOIN " - + UserMetaMapper.USER_TABLE_NAME - + " ut ON ut.user_id = ot.owner_id" - + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" - + " ot.metadata_object_type = #{metadataObjectType} AND" - + " ot.owner_type = 'USER' AND" - + " ot.deleted_at = 0 AND ut.deleted_at = 0") + @SelectProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "selectUserOwnerMetaByMetadataObjectIdAndType") UserPO selectUserOwnerMetaByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Select( - "SELECT gt.group_id as groupId," - + " gt.group_name as groupName," - + " gt.metalake_id as metalakeId," - + " gt.audit_info as auditInfo," - + " gt.current_version as currentVersion," - + " gt.last_version as lastVersion," - + " gt.deleted_at as deletedAt" - + " FROM " - + OWNER_TABLE_NAME - + " ot JOIN " - + GroupMetaMapper.GROUP_TABLE_NAME - + " gt ON gt.group_id = ot.owner_id" - + " WHERE ot.metadata_object_id = #{metadataObjectId} AND" - + " ot.metadata_object_type = #{metadataObjectType} AND" - + " ot.owner_type = 'GROUP' AND" - + " ot.deleted_at = 0 AND gt.deleted_at = 0") + @SelectProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "selectGroupOwnerMetaByMetadataObjectIdAndType") GroupPO selectGroupOwnerMetaByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Insert( - "INSERT INTO " - + OWNER_TABLE_NAME - + "(metalake_id, metadata_object_id, metadata_object_type, owner_id, owner_type," - + " audit_info, current_version, last_version, deleted_at)" - + " VALUES (" - + " #{ownerRelPO.metalakeId}," - + " #{ownerRelPO.metadataObjectId}," - + " #{ownerRelPO.metadataObjectType}," - + " #{ownerRelPO.ownerId}," - + " #{ownerRelPO.ownerType}," - + " #{ownerRelPO.auditInfo}," - + " #{ownerRelPO.currentVersion}," - + " #{ownerRelPO.lastVersion}," - + " #{ownerRelPO.deletedAt}" - + ")") + @InsertProvider(type = OwnerMetaSQLProviderFactory.class, method = "insertOwnerRel") void insertOwnerRel(@Param("ownerRelPO") OwnerRelPO ownerRelPO); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metadata_object_id = #{metadataObjectId} AND metadata_object_type = #{metadataObjectType} AND deleted_at = 0") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByMetadataObjectIdAndType") void softDeleteOwnerRelByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE owner_id = #{ownerId} AND owner_type = #{ownerType} AND deleted_at = 0") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByOwnerIdAndType") void softDeleteOwnerRelByOwnerIdAndType( @Param("ownerId") Long ownerId, @Param("ownerType") String ownerType); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at =0") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByMetalakeId") void softDeleteOwnerRelByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE EXISTS (" - + " SELECT ct.catalog_id FROM " - + CatalogMetaMapper.TABLE_NAME - + " ct WHERE ct.catalog_id = #{catalogId} AND ct.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'" - + " UNION " - + " SELECT st.catalog_id FROM " - + SchemaMetaMapper.TABLE_NAME - + " st WHERE st.catalog_id = #{catalogId} AND st.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" - + " UNION " - + " SELECT tt.catalog_id FROM " - + TopicMetaMapper.TABLE_NAME - + " tt WHERE tt.catalog_id = #{catalogId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" - + " UNION " - + " SELECT tat.catalog_id FROM " - + TableMetaMapper.TABLE_NAME - + " tat WHERE tat.catalog_id = #{catalogId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" - + " UNION " - + " SELECT ft.catalog_id FROM " - + FilesetMetaMapper.META_TABLE_NAME - + " ft WHERE ft.catalog_id = #{catalogId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND" - + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" - + ")") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "softDeleteOwnerRelByCatalogId") void softDeleteOwnerRelByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + OWNER_TABLE_NAME - + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE EXISTS (" - + " SELECT st.schema_id FROM " - + SchemaMetaMapper.TABLE_NAME - + " st WHERE st.schema_id = #{schemaId} AND st.deleted_at = 0 AND ot.deleted_at = 0 " - + "AND st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" - + " UNION " - + " SELECT tt.schema_id FROM " - + TopicMetaMapper.TABLE_NAME - + " tt WHERE tt.schema_id = #{schemaId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" - + " UNION " - + " SELECT tat.schema_id FROM " - + TableMetaMapper.TABLE_NAME - + " tat WHERE tat.schema_id = #{schemaId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" - + " UNION " - + " SELECT ft.schema_id FROM " - + FilesetMetaMapper.META_TABLE_NAME - + " ft WHERE ft.schema_id = #{schemaId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND " - + "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" - + ")") - void sotDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId); + @UpdateProvider(type = OwnerMetaSQLProviderFactory.class, method = "softDeleteOwnerRelBySchemaId") + void softDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId); - @Delete( - "DELETE FROM " - + OWNER_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @UpdateProvider( + type = OwnerMetaSQLProviderFactory.class, + method = "deleteOwnerMetasByLegacyTimeline") Integer deleteOwnerMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java new file mode 100644 index 00000000000..8ddf53d5d50 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/OwnerMetaSQLProviderFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.OwnerMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.OwnerRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class OwnerMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new OwnerMetaMySQLProvider(), + JDBCBackendType.H2, new OwnerMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new OwnerMetaPostgreSQLProvider()); + + public static OwnerMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class OwnerMetaMySQLProvider extends OwnerMetaBaseSQLProvider {} + + static class OwnerMetaH2Provider extends OwnerMetaBaseSQLProvider {} + + public static String selectUserOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider() + .selectUserOwnerMetaByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String selectGroupOwnerMetaByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider() + .selectGroupOwnerMetaByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String insertOwnerRel(@Param("ownerRelPO") OwnerRelPO ownerRelPO) { + return getProvider().insertOwnerRel(ownerRelPO); + } + + public static String softDeleteOwnerRelByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider() + .softDeleteOwnerRelByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String softDeleteOwnerRelByOwnerIdAndType( + @Param("ownerId") Long ownerId, @Param("ownerType") String ownerType) { + return getProvider().softDeleteOwnerRelByOwnerIdAndType(ownerId, ownerType); + } + + public static String softDeleteOwnerRelByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteOwnerRelByMetalakeId(metalakeId); + } + + public static String softDeleteOwnerRelByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteOwnerRelByCatalogId(catalogId); + } + + public static String softDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().sotDeleteOwnerRelBySchemaId(schemaId); + } + + public static String deleteOwnerMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteOwnerMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaBaseSQLProvider.java new file mode 100644 index 00000000000..dad878842a3 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaBaseSQLProvider.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.GROUP_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.ROLE_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.USER_ROLE_RELATION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.RolePO; +import org.apache.ibatis.annotations.Param; + +public class RoleMetaBaseSQLProvider { + + public String selectRoleMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String roleName) { + return "SELECT role_id as roleId, role_name as roleName," + + " metalake_id as metalakeId, properties as properties," + + " audit_info as auditInfo, current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" + + " AND deleted_at = 0"; + } + + public String selectRoleIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String name) { + return "SELECT role_id as roleId FROM " + + ROLE_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" + + " AND deleted_at = 0"; + } + + public String listRolesByUserId(@Param("userId") Long userId) { + return "SELECT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + USER_ROLE_RELATION_TABLE_NAME + + " re ON ro.role_id = re.role_id" + + " WHERE re.user_id = #{userId}" + + " AND ro.deleted_at = 0 AND re.deleted_at = 0"; + } + + public String listRolesByGroupId(Long groupId) { + return "SELECT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + GROUP_ROLE_RELATION_TABLE_NAME + + " ge ON ro.role_id = ge.role_id" + + " WHERE ge.group_id = #{groupId}" + + " AND ro.deleted_at = 0 AND ge.deleted_at = 0"; + } + + public String listRolesByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT DISTINCT ro.role_id as roleId, ro.role_name as roleName," + + " ro.metalake_id as metalakeId, ro.properties as properties," + + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + + " FROM " + + ROLE_TABLE_NAME + + " ro JOIN " + + SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME + + " se ON ro.role_id = se.role_id" + + " WHERE se.metadata_object_id = #{metadataObjectId}" + + " AND se.type = #{metadataObjectType}" + + " AND ro.deleted_at = 0 AND se.deleted_at = 0"; + } + + public String insertRoleMeta(@Param("roleMeta") RolePO rolePO) { + return "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " )"; + } + + public String insertRoleMetaOnDuplicateKeyUpdate(@Param("roleMeta") RolePO rolePO) { + return "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " ) ON DUPLICATE KEY UPDATE" + + " role_name = #{roleMeta.roleName}," + + " metalake_id = #{roleMeta.metalakeId}," + + " properties = #{roleMeta.properties}," + + " audit_info = #{roleMeta.auditInfo}," + + " current_version = #{roleMeta.currentVersion}," + + " last_version = #{roleMeta.lastVersion}," + + " deleted_at = #{roleMeta.deletedAt}"; + } + + public String softDeleteRoleMetaByRoleId(Long roleId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteRoleMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + ROLE_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java index 2c9190d69d6..d7478c3b7f4 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.RolePO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -40,134 +40,50 @@ public interface RoleMetaMapper { String USER_ROLE_RELATION_TABLE_NAME = "user_role_rel"; String GROUP_ROLE_RELATION_TABLE_NAME = "group_role_rel"; - @Select( - "SELECT role_id as roleId, role_name as roleName," - + " metalake_id as metalakeId, properties as properties," - + " audit_info as auditInfo, current_version as currentVersion," - + " last_version as lastVersion, deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" - + " AND deleted_at = 0") + @SelectProvider( + type = RoleMetaSQLProviderFactory.class, + method = "selectRoleMetaByMetalakeIdAndName") RolePO selectRoleMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("roleName") String roleName); - @Select( - "SELECT role_id as roleId FROM " - + ROLE_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND role_name = #{roleName}" - + " AND deleted_at = 0") + @SelectProvider( + type = RoleMetaSQLProviderFactory.class, + method = "selectRoleIdByMetalakeIdAndName") Long selectRoleIdByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("roleName") String name); - @Select( - "SELECT ro.role_id as roleId, ro.role_name as roleName," - + " ro.metalake_id as metalakeId, ro.properties as properties," - + " ro.audit_info as auditInfo, ro.current_version as currentVersion," - + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " ro JOIN " - + USER_ROLE_RELATION_TABLE_NAME - + " re ON ro.role_id = re.role_id" - + " WHERE re.user_id = #{userId}" - + " AND ro.deleted_at = 0 AND re.deleted_at = 0") + @SelectProvider(type = RoleMetaSQLProviderFactory.class, method = "listRolesByUserId") List listRolesByUserId(@Param("userId") Long userId); - @Select( - "SELECT ro.role_id as roleId, ro.role_name as roleName," - + " ro.metalake_id as metalakeId, ro.properties as properties," - + " ro.audit_info as auditInfo, ro.current_version as currentVersion," - + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " ro JOIN " - + GROUP_ROLE_RELATION_TABLE_NAME - + " ge ON ro.role_id = ge.role_id" - + " WHERE ge.group_id = #{groupId}" - + " AND ro.deleted_at = 0 AND ge.deleted_at = 0") + @SelectProvider(type = RoleMetaSQLProviderFactory.class, method = "listRolesByGroupId") List listRolesByGroupId(Long groupId); - @Select( - "SELECT DISTINCT ro.role_id as roleId, ro.role_name as roleName," - + " ro.metalake_id as metalakeId, ro.properties as properties," - + " ro.audit_info as auditInfo, ro.current_version as currentVersion," - + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" - + " FROM " - + ROLE_TABLE_NAME - + " ro JOIN " - + SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME - + " se ON ro.role_id = se.role_id" - + " WHERE se.metadata_object_id = #{metadataObjectId}" - + " AND se.type = #{metadataObjectType}" - + " AND ro.deleted_at = 0 AND se.deleted_at = 0") + @SelectProvider( + type = RoleMetaSQLProviderFactory.class, + method = "listRolesByMetadataObjectIdAndType") List listRolesByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Insert( - "INSERT INTO " - + ROLE_TABLE_NAME - + "(role_id, role_name," - + " metalake_id, properties," - + " audit_info, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{roleMeta.roleId}," - + " #{roleMeta.roleName}," - + " #{roleMeta.metalakeId}," - + " #{roleMeta.properties}," - + " #{roleMeta.auditInfo}," - + " #{roleMeta.currentVersion}," - + " #{roleMeta.lastVersion}," - + " #{roleMeta.deletedAt}" - + " )") + @InsertProvider(type = RoleMetaSQLProviderFactory.class, method = "insertRoleMeta") void insertRoleMeta(@Param("roleMeta") RolePO rolePO); - @Insert( - "INSERT INTO " - + ROLE_TABLE_NAME - + "(role_id, role_name," - + " metalake_id, properties," - + " audit_info, current_version, last_version, deleted_at)" - + " VALUES(" - + " #{roleMeta.roleId}," - + " #{roleMeta.roleName}," - + " #{roleMeta.metalakeId}," - + " #{roleMeta.properties}," - + " #{roleMeta.auditInfo}," - + " #{roleMeta.currentVersion}," - + " #{roleMeta.lastVersion}," - + " #{roleMeta.deletedAt}" - + " ) ON DUPLICATE KEY UPDATE" - + " role_name = #{roleMeta.roleName}," - + " metalake_id = #{roleMeta.metalakeId}," - + " properties = #{roleMeta.properties}," - + " audit_info = #{roleMeta.auditInfo}," - + " current_version = #{roleMeta.currentVersion}," - + " last_version = #{roleMeta.lastVersion}," - + " deleted_at = #{roleMeta.deletedAt}") + @InsertProvider( + type = RoleMetaSQLProviderFactory.class, + method = "insertRoleMetaOnDuplicateKeyUpdate") void insertRoleMetaOnDuplicateKeyUpdate(@Param("roleMeta") RolePO rolePO); - @Update( - "UPDATE " - + ROLE_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider(type = RoleMetaSQLProviderFactory.class, method = "softDeleteRoleMetaByRoleId") void softDeleteRoleMetaByRoleId(Long roleId); - @Update( - "UPDATE " - + ROLE_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = RoleMetaSQLProviderFactory.class, + method = "softDeleteRoleMetasByMetalakeId") void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + ROLE_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = RoleMetaSQLProviderFactory.class, + method = "deleteRoleMetasByLegacyTimeline") Integer deleteRoleMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java new file mode 100644 index 00000000000..41599386003 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/RoleMetaSQLProviderFactory.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.RoleMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.RolePO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class RoleMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new RoleMetaMySQLProvider(), + JDBCBackendType.H2, new RoleMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new RoleMetaPostgreSQLProvider()); + + public static RoleMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class RoleMetaMySQLProvider extends RoleMetaBaseSQLProvider {} + + static class RoleMetaH2Provider extends RoleMetaBaseSQLProvider {} + + public static String selectRoleMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String roleName) { + return getProvider().selectRoleMetaByMetalakeIdAndName(metalakeId, roleName); + } + + public static String selectRoleIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("roleName") String name) { + return getProvider().selectRoleIdByMetalakeIdAndName(metalakeId, name); + } + + public static String listRolesByUserId(@Param("userId") Long userId) { + return getProvider().listRolesByUserId(userId); + } + + public static String listRolesByGroupId(Long groupId) { + return getProvider().listRolesByGroupId(groupId); + } + + public static String listRolesByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String type) { + return getProvider().listRolesByMetadataObjectIdAndType(metadataObjectId, type); + } + + public static String insertRoleMeta(@Param("roleMeta") RolePO rolePO) { + return getProvider().insertRoleMeta(rolePO); + } + + public static String insertRoleMetaOnDuplicateKeyUpdate(@Param("roleMeta") RolePO rolePO) { + return getProvider().insertRoleMetaOnDuplicateKeyUpdate(rolePO); + } + + public static String softDeleteRoleMetaByRoleId(Long roleId) { + return getProvider().softDeleteRoleMetaByRoleId(roleId); + } + + public static String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteRoleMetasByMetalakeId(metalakeId); + } + + public static String deleteRoleMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteRoleMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaBaseSQLProvider.java new file mode 100644 index 00000000000..056b8d2b581 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaBaseSQLProvider.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.SchemaPO; +import org.apache.ibatis.annotations.Param; + +public class SchemaMetaBaseSQLProvider { + public String listSchemaPOsByCatalogId(@Param("catalogId") Long catalogId) { + return "SELECT schema_id as schemaId, schema_name as schemaName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_comment as schemaComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String selectSchemaIdByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return "SELECT schema_id as schemaId FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName}" + + " AND deleted_at = 0"; + } + + public String selectSchemaMetaByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return "SELECT schema_id as schemaId, schema_name as schemaName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_comment as schemaComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName} AND deleted_at = 0"; + } + + public String selectSchemaMetaById(@Param("schemaId") Long schemaId) { + return "SELECT schema_id as schemaId, schema_name as schemaName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_comment as schemaComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String insertSchemaMeta(@Param("schemaMeta") SchemaPO schemaPO) { + return "INSERT INTO " + + TABLE_NAME + + "(schema_id, schema_name, metalake_id," + + " catalog_id, schema_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{schemaMeta.schemaId}," + + " #{schemaMeta.schemaName}," + + " #{schemaMeta.metalakeId}," + + " #{schemaMeta.catalogId}," + + " #{schemaMeta.schemaComment}," + + " #{schemaMeta.properties}," + + " #{schemaMeta.auditInfo}," + + " #{schemaMeta.currentVersion}," + + " #{schemaMeta.lastVersion}," + + " #{schemaMeta.deletedAt}" + + " )"; + } + + public String insertSchemaMetaOnDuplicateKeyUpdate(@Param("schemaMeta") SchemaPO schemaPO) { + return "INSERT INTO " + + TABLE_NAME + + "(schema_id, schema_name, metalake_id," + + " catalog_id, schema_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{schemaMeta.schemaId}," + + " #{schemaMeta.schemaName}," + + " #{schemaMeta.metalakeId}," + + " #{schemaMeta.catalogId}," + + " #{schemaMeta.schemaComment}," + + " #{schemaMeta.properties}," + + " #{schemaMeta.auditInfo}," + + " #{schemaMeta.currentVersion}," + + " #{schemaMeta.lastVersion}," + + " #{schemaMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " schema_name = #{schemaMeta.schemaName}," + + " metalake_id = #{schemaMeta.metalakeId}," + + " catalog_id = #{schemaMeta.catalogId}," + + " schema_comment = #{schemaMeta.schemaComment}," + + " properties = #{schemaMeta.properties}," + + " audit_info = #{schemaMeta.auditInfo}," + + " current_version = #{schemaMeta.currentVersion}," + + " last_version = #{schemaMeta.lastVersion}," + + " deleted_at = #{schemaMeta.deletedAt}"; + } + + public String updateSchemaMeta( + @Param("newSchemaMeta") SchemaPO newSchemaPO, @Param("oldSchemaMeta") SchemaPO oldSchemaPO) { + return "UPDATE " + + TABLE_NAME + + " SET schema_name = #{newSchemaMeta.schemaName}," + + " metalake_id = #{newSchemaMeta.metalakeId}," + + " catalog_id = #{newSchemaMeta.catalogId}," + + " schema_comment = #{newSchemaMeta.schemaComment}," + + " properties = #{newSchemaMeta.properties}," + + " audit_info = #{newSchemaMeta.auditInfo}," + + " current_version = #{newSchemaMeta.currentVersion}," + + " last_version = #{newSchemaMeta.lastVersion}," + + " deleted_at = #{newSchemaMeta.deletedAt}" + + " WHERE schema_id = #{oldSchemaMeta.schemaId}" + + " AND schema_name = #{oldSchemaMeta.schemaName}" + + " AND metalake_id = #{oldSchemaMeta.metalakeId}" + + " AND catalog_id = #{oldSchemaMeta.catalogId}" + + " AND (schema_comment IS NULL OR schema_comment = #{oldSchemaMeta.schemaComment})" + + " AND properties = #{oldSchemaMeta.properties}" + + " AND audit_info = #{oldSchemaMeta.auditInfo}" + + " AND current_version = #{oldSchemaMeta.currentVersion}" + + " AND last_version = #{oldSchemaMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteSchemaMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String softDeleteSchemaMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String deleteSchemaMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java index a71a4b7b6f3..054248dac0c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.SchemaPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for schema meta operation SQLs. @@ -38,151 +38,54 @@ public interface SchemaMetaMapper { String TABLE_NAME = "schema_meta"; - @Select( - "SELECT schema_id as schemaId, schema_name as schemaName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_comment as schemaComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @SelectProvider(type = SchemaMetaSQLProviderFactory.class, method = "listSchemaPOsByCatalogId") List listSchemaPOsByCatalogId(@Param("catalogId") Long catalogId); - @Select( - "SELECT schema_id as schemaId FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName}" - + " AND deleted_at = 0") + @SelectProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "selectSchemaIdByCatalogIdAndName") Long selectSchemaIdByCatalogIdAndName( @Param("catalogId") Long catalogId, @Param("schemaName") String name); - @Select( - "SELECT schema_id as schemaId, schema_name as schemaName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_comment as schemaComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE catalog_id = #{catalogId} AND schema_name = #{schemaName} AND deleted_at = 0") + @SelectProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "selectSchemaMetaByCatalogIdAndName") SchemaPO selectSchemaMetaByCatalogIdAndName( @Param("catalogId") Long catalogId, @Param("schemaName") String name); - @Select( - "SELECT schema_id as schemaId, schema_name as schemaName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_comment as schemaComment, properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @SelectProvider(type = SchemaMetaSQLProviderFactory.class, method = "selectSchemaMetaById") SchemaPO selectSchemaMetaById(@Param("schemaId") Long schemaId); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(schema_id, schema_name, metalake_id," - + " catalog_id, schema_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{schemaMeta.schemaId}," - + " #{schemaMeta.schemaName}," - + " #{schemaMeta.metalakeId}," - + " #{schemaMeta.catalogId}," - + " #{schemaMeta.schemaComment}," - + " #{schemaMeta.properties}," - + " #{schemaMeta.auditInfo}," - + " #{schemaMeta.currentVersion}," - + " #{schemaMeta.lastVersion}," - + " #{schemaMeta.deletedAt}" - + " )") + @InsertProvider(type = SchemaMetaSQLProviderFactory.class, method = "insertSchemaMeta") void insertSchemaMeta(@Param("schemaMeta") SchemaPO schemaPO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(schema_id, schema_name, metalake_id," - + " catalog_id, schema_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{schemaMeta.schemaId}," - + " #{schemaMeta.schemaName}," - + " #{schemaMeta.metalakeId}," - + " #{schemaMeta.catalogId}," - + " #{schemaMeta.schemaComment}," - + " #{schemaMeta.properties}," - + " #{schemaMeta.auditInfo}," - + " #{schemaMeta.currentVersion}," - + " #{schemaMeta.lastVersion}," - + " #{schemaMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " schema_name = #{schemaMeta.schemaName}," - + " metalake_id = #{schemaMeta.metalakeId}," - + " catalog_id = #{schemaMeta.catalogId}," - + " schema_comment = #{schemaMeta.schemaComment}," - + " properties = #{schemaMeta.properties}," - + " audit_info = #{schemaMeta.auditInfo}," - + " current_version = #{schemaMeta.currentVersion}," - + " last_version = #{schemaMeta.lastVersion}," - + " deleted_at = #{schemaMeta.deletedAt}") + @InsertProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "insertSchemaMetaOnDuplicateKeyUpdate") void insertSchemaMetaOnDuplicateKeyUpdate(@Param("schemaMeta") SchemaPO schemaPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET schema_name = #{newSchemaMeta.schemaName}," - + " metalake_id = #{newSchemaMeta.metalakeId}," - + " catalog_id = #{newSchemaMeta.catalogId}," - + " schema_comment = #{newSchemaMeta.schemaComment}," - + " properties = #{newSchemaMeta.properties}," - + " audit_info = #{newSchemaMeta.auditInfo}," - + " current_version = #{newSchemaMeta.currentVersion}," - + " last_version = #{newSchemaMeta.lastVersion}," - + " deleted_at = #{newSchemaMeta.deletedAt}" - + " WHERE schema_id = #{oldSchemaMeta.schemaId}" - + " AND schema_name = #{oldSchemaMeta.schemaName}" - + " AND metalake_id = #{oldSchemaMeta.metalakeId}" - + " AND catalog_id = #{oldSchemaMeta.catalogId}" - + " AND (schema_comment IS NULL OR schema_comment = #{oldSchemaMeta.schemaComment})" - + " AND properties = #{oldSchemaMeta.properties}" - + " AND audit_info = #{oldSchemaMeta.auditInfo}" - + " AND current_version = #{oldSchemaMeta.currentVersion}" - + " AND last_version = #{oldSchemaMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = SchemaMetaSQLProviderFactory.class, method = "updateSchemaMeta") Integer updateSchemaMeta( @Param("newSchemaMeta") SchemaPO newSchemaPO, @Param("oldSchemaMeta") SchemaPO oldSchemaPO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "softDeleteSchemaMetasBySchemaId") Integer softDeleteSchemaMetasBySchemaId(@Param("schemaId") Long schemaId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "softDeleteSchemaMetasByMetalakeId") Integer softDeleteSchemaMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "softDeleteSchemaMetasByCatalogId") Integer softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "deleteSchemaMetasByLegacyTimeline") Integer deleteSchemaMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java new file mode 100644 index 00000000000..87f636e61d8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.SchemaMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.SchemaPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class SchemaMetaSQLProviderFactory { + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new SchemaMetaMySQLProvider(), + JDBCBackendType.H2, new SchemaMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new SchemaMetaPostgreSQLProvider()); + + public static SchemaMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class SchemaMetaMySQLProvider extends SchemaMetaBaseSQLProvider {} + + static class SchemaMetaH2Provider extends SchemaMetaBaseSQLProvider {} + + public static String listSchemaPOsByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().listSchemaPOsByCatalogId(catalogId); + } + + public static String selectSchemaIdByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return getProvider().selectSchemaIdByCatalogIdAndName(catalogId, name); + } + + public static String selectSchemaMetaByCatalogIdAndName( + @Param("catalogId") Long catalogId, @Param("schemaName") String name) { + return getProvider().selectSchemaMetaByCatalogIdAndName(catalogId, name); + } + + public static String selectSchemaMetaById(@Param("schemaId") Long schemaId) { + return getProvider().selectSchemaMetaById(schemaId); + } + + public static String insertSchemaMeta(@Param("schemaMeta") SchemaPO schemaPO) { + return getProvider().insertSchemaMeta(schemaPO); + } + + public static String insertSchemaMetaOnDuplicateKeyUpdate( + @Param("schemaMeta") SchemaPO schemaPO) { + return getProvider().insertSchemaMetaOnDuplicateKeyUpdate(schemaPO); + } + + public static String updateSchemaMeta( + @Param("newSchemaMeta") SchemaPO newSchemaPO, @Param("oldSchemaMeta") SchemaPO oldSchemaPO) { + return getProvider().updateSchemaMeta(newSchemaPO, oldSchemaPO); + } + + public static String softDeleteSchemaMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteSchemaMetasBySchemaId(schemaId); + } + + public static String softDeleteSchemaMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteSchemaMetasByMetalakeId(metalakeId); + } + + public static String softDeleteSchemaMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteSchemaMetasByCatalogId(catalogId); + } + + public static String deleteSchemaMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteSchemaMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectBaseSQLProvider.java new file mode 100644 index 00000000000..7e0a3674642 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectBaseSQLProvider.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.ROLE_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.SecurableObjectPO; +import org.apache.ibatis.annotations.Param; + +public class SecurableObjectBaseSQLProvider { + + public String batchInsertSecurableObjects( + @Param("securableObjects") List securableObjectPOs) { + return ""; + } + + public String softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " ob SET ob.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE exists (SELECT * from " + + ROLE_TABLE_NAME + + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" + + " AND ro.deleted_at = 0) AND ob.deleted_at = 0"; + } + + public String listSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return "SELECT role_id as roleId, metadata_object_id as metadataObjectId," + + " type as type, privilege_names as privilegeNames," + + " privilege_conditions as privilegeConditions, current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String deleteSecurableObjectsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java index 9a8ae52c309..e5fa90cc914 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java @@ -20,11 +20,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.SecurableObjectPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -39,61 +39,30 @@ public interface SecurableObjectMapper { String SECURABLE_OBJECT_TABLE_NAME = "role_meta_securable_object"; String ROLE_TABLE_NAME = "role_meta"; - @Insert({ - "" - }) + @InsertProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "batchInsertSecurableObjects") void batchInsertSecurableObjects( @Param("securableObjects") List securableObjectPOs); - @Update( - "UPDATE " - + SECURABLE_OBJECT_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "softDeleteSecurableObjectsByRoleId") void softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId); - @Update( - "UPDATE " - + SECURABLE_OBJECT_TABLE_NAME - + " ob SET ob.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE exists (SELECT * from " - + ROLE_TABLE_NAME - + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" - + " AND ro.deleted_at = 0) AND ob.deleted_at = 0") + @UpdateProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "softDeleteRoleMetasByMetalakeId") void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Select( - "SELECT role_id as roleId, metadata_object_id as metadataObjectId," - + " type as type, privilege_names as privilegeNames," - + " privilege_conditions as privilegeConditions, current_version as currentVersion," - + " last_version as lastVersion, deleted_at as deletedAt" - + " FROM " - + SECURABLE_OBJECT_TABLE_NAME - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @SelectProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "listSecurableObjectsByRoleId") List listSecurableObjectsByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + SECURABLE_OBJECT_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = SecurableObjectSQLProviderFactory.class, + method = "deleteSecurableObjectsByLegacyTimeline") Integer deleteSecurableObjectsByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java new file mode 100644 index 00000000000..da4ddc730a0 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.SecurableObjectPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.SecurableObjectPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class SecurableObjectSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new SecurableObjectMySQLProvider(), + JDBCBackendType.H2, new SecurableObjectH2Provider(), + JDBCBackendType.POSTGRESQL, new SecurableObjectPostgreSQLProvider()); + + public static SecurableObjectBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class SecurableObjectMySQLProvider extends SecurableObjectBaseSQLProvider {} + + static class SecurableObjectH2Provider extends SecurableObjectBaseSQLProvider {} + + public static String batchInsertSecurableObjects( + @Param("securableObjects") List securableObjectPOs) { + return getProvider().batchInsertSecurableObjects(securableObjectPOs); + } + + public static String softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return getProvider().softDeleteSecurableObjectsByRoleId(roleId); + } + + public static String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteRoleMetasByMetalakeId(metalakeId); + } + + public static String listSecurableObjectsByRoleId(@Param("roleId") Long roleId) { + return getProvider().listSecurableObjectsByRoleId(roleId); + } + + public static String deleteSecurableObjectsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteSecurableObjectsByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaBaseSQLProvider.java new file mode 100644 index 00000000000..03cf2582ff6 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaBaseSQLProvider.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.TablePO; +import org.apache.ibatis.annotations.Param; + +public class TableMetaBaseSQLProvider { + + public String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) { + return "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String selectTableIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return "SELECT table_id as tableId FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND table_name = #{tableName}" + + " AND deleted_at = 0"; + } + + public String selectTableMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND table_name = #{tableName} AND deleted_at = 0"; + } + + public String selectTableMetaById(@Param("tableId") Long tableId) { + return "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE table_id = #{tableId} AND deleted_at = 0"; + } + + public String insertTableMeta(@Param("tableMeta") TablePO tablePO) { + return "INSERT INTO " + + TABLE_NAME + + "(table_id, table_name, metalake_id," + + " catalog_id, schema_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tableMeta.tableId}," + + " #{tableMeta.tableName}," + + " #{tableMeta.metalakeId}," + + " #{tableMeta.catalogId}," + + " #{tableMeta.schemaId}," + + " #{tableMeta.auditInfo}," + + " #{tableMeta.currentVersion}," + + " #{tableMeta.lastVersion}," + + " #{tableMeta.deletedAt}" + + " )"; + } + + public String insertTableMetaOnDuplicateKeyUpdate(@Param("tableMeta") TablePO tablePO) { + return "INSERT INTO " + + TABLE_NAME + + "(table_id, table_name, metalake_id," + + " catalog_id, schema_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tableMeta.tableId}," + + " #{tableMeta.tableName}," + + " #{tableMeta.metalakeId}," + + " #{tableMeta.catalogId}," + + " #{tableMeta.schemaId}," + + " #{tableMeta.auditInfo}," + + " #{tableMeta.currentVersion}," + + " #{tableMeta.lastVersion}," + + " #{tableMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " table_name = #{tableMeta.tableName}," + + " metalake_id = #{tableMeta.metalakeId}," + + " catalog_id = #{tableMeta.catalogId}," + + " schema_id = #{tableMeta.schemaId}," + + " audit_info = #{tableMeta.auditInfo}," + + " current_version = #{tableMeta.currentVersion}," + + " last_version = #{tableMeta.lastVersion}," + + " deleted_at = #{tableMeta.deletedAt}"; + } + + public String updateTableMeta( + @Param("newTableMeta") TablePO newTablePO, @Param("oldTableMeta") TablePO oldTablePO) { + return "UPDATE " + + TABLE_NAME + + " SET table_name = #{newTableMeta.tableName}," + + " metalake_id = #{newTableMeta.metalakeId}," + + " catalog_id = #{newTableMeta.catalogId}," + + " schema_id = #{newTableMeta.schemaId}," + + " audit_info = #{newTableMeta.auditInfo}," + + " current_version = #{newTableMeta.currentVersion}," + + " last_version = #{newTableMeta.lastVersion}," + + " deleted_at = #{newTableMeta.deletedAt}" + + " WHERE table_id = #{oldTableMeta.tableId}" + + " AND table_name = #{oldTableMeta.tableName}" + + " AND metalake_id = #{oldTableMeta.metalakeId}" + + " AND catalog_id = #{oldTableMeta.catalogId}" + + " AND schema_id = #{oldTableMeta.schemaId}" + + " AND audit_info = #{oldTableMeta.auditInfo}" + + " AND current_version = #{oldTableMeta.currentVersion}" + + " AND last_version = #{oldTableMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteTableMetasByTableId(@Param("tableId") Long tableId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE table_id = #{tableId} AND deleted_at = 0"; + } + + public String softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String deleteTableMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java index 98edc595fd7..a5224593765 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TablePO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -38,154 +38,59 @@ public interface TableMetaMapper { String TABLE_NAME = "table_meta"; - @Select( - "SELECT table_id as tableId, table_name as tableName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_id as schemaId, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsBySchemaId") List listTablePOsBySchemaId(@Param("schemaId") Long schemaId); - @Select( - "SELECT table_id as tableId FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND table_name = #{tableName}" - + " AND deleted_at = 0") + @SelectProvider( + type = TableMetaSQLProviderFactory.class, + method = "selectTableIdBySchemaIdAndName") Long selectTableIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name); - @Select( - "SELECT table_id as tableId, table_name as tableName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_id as schemaId, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND table_name = #{tableName} AND deleted_at = 0") + @SelectProvider( + type = TableMetaSQLProviderFactory.class, + method = "selectTableMetaBySchemaIdAndName") TablePO selectTableMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name); - @Select( - "SELECT table_id as tableId, table_name as tableName," - + " metalake_id as metalakeId, catalog_id as catalogId," - + " schema_id as schemaId, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE table_id = #{tableId} AND deleted_at = 0") + @SelectProvider(type = TableMetaSQLProviderFactory.class, method = "selectTableMetaById") TablePO selectTableMetaById(@Param("tableId") Long tableId); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(table_id, table_name, metalake_id," - + " catalog_id, schema_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tableMeta.tableId}," - + " #{tableMeta.tableName}," - + " #{tableMeta.metalakeId}," - + " #{tableMeta.catalogId}," - + " #{tableMeta.schemaId}," - + " #{tableMeta.auditInfo}," - + " #{tableMeta.currentVersion}," - + " #{tableMeta.lastVersion}," - + " #{tableMeta.deletedAt}" - + " )") + @InsertProvider(type = TableMetaSQLProviderFactory.class, method = "insertTableMeta") void insertTableMeta(@Param("tableMeta") TablePO tablePO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(table_id, table_name, metalake_id," - + " catalog_id, schema_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tableMeta.tableId}," - + " #{tableMeta.tableName}," - + " #{tableMeta.metalakeId}," - + " #{tableMeta.catalogId}," - + " #{tableMeta.schemaId}," - + " #{tableMeta.auditInfo}," - + " #{tableMeta.currentVersion}," - + " #{tableMeta.lastVersion}," - + " #{tableMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " table_name = #{tableMeta.tableName}," - + " metalake_id = #{tableMeta.metalakeId}," - + " catalog_id = #{tableMeta.catalogId}," - + " schema_id = #{tableMeta.schemaId}," - + " audit_info = #{tableMeta.auditInfo}," - + " current_version = #{tableMeta.currentVersion}," - + " last_version = #{tableMeta.lastVersion}," - + " deleted_at = #{tableMeta.deletedAt}") + @InsertProvider( + type = TableMetaSQLProviderFactory.class, + method = "insertTableMetaOnDuplicateKeyUpdate") void insertTableMetaOnDuplicateKeyUpdate(@Param("tableMeta") TablePO tablePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET table_name = #{newTableMeta.tableName}," - + " metalake_id = #{newTableMeta.metalakeId}," - + " catalog_id = #{newTableMeta.catalogId}," - + " schema_id = #{newTableMeta.schemaId}," - + " audit_info = #{newTableMeta.auditInfo}," - + " current_version = #{newTableMeta.currentVersion}," - + " last_version = #{newTableMeta.lastVersion}," - + " deleted_at = #{newTableMeta.deletedAt}" - + " WHERE table_id = #{oldTableMeta.tableId}" - + " AND table_name = #{oldTableMeta.tableName}" - + " AND metalake_id = #{oldTableMeta.metalakeId}" - + " AND catalog_id = #{oldTableMeta.catalogId}" - + " AND schema_id = #{oldTableMeta.schemaId}" - + " AND audit_info = #{oldTableMeta.auditInfo}" - + " AND current_version = #{oldTableMeta.currentVersion}" - + " AND last_version = #{oldTableMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = TableMetaSQLProviderFactory.class, method = "updateTableMeta") Integer updateTableMeta( @Param("newTableMeta") TablePO newTablePO, @Param("oldTableMeta") TablePO oldTablePO); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE table_id = #{tableId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasByTableId") Integer softDeleteTableMetasByTableId(@Param("tableId") Long tableId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasByMetalakeId") Integer softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasByCatalogId") Integer softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = TableMetaSQLProviderFactory.class, + method = "softDeleteTableMetasBySchemaId") Integer softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = TableMetaSQLProviderFactory.class, + method = "deleteTableMetasByLegacyTimeline") Integer deleteTableMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java new file mode 100644 index 00000000000..619648c1f52 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TableMetaSQLProviderFactory.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TableMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.TablePO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TableMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TableMetaMySQLProvider(), + JDBCBackendType.H2, new TableMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new TableMetaPostgreSQLProvider()); + + public static TableMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TableMetaMySQLProvider extends TableMetaBaseSQLProvider {} + + static class TableMetaH2Provider extends TableMetaBaseSQLProvider {} + + public static String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().listTablePOsBySchemaId(schemaId); + } + + public static String selectTableIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return getProvider().selectTableIdBySchemaIdAndName(schemaId, name); + } + + public static String selectTableMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("tableName") String name) { + return getProvider().selectTableMetaBySchemaIdAndName(schemaId, name); + } + + public static String selectTableMetaById(@Param("tableId") Long tableId) { + return getProvider().selectTableMetaById(tableId); + } + + public static String insertTableMeta(@Param("tableMeta") TablePO tablePO) { + return getProvider().insertTableMeta(tablePO); + } + + public static String insertTableMetaOnDuplicateKeyUpdate(@Param("tableMeta") TablePO tablePO) { + return getProvider().insertTableMetaOnDuplicateKeyUpdate(tablePO); + } + + public static String updateTableMeta( + @Param("newTableMeta") TablePO newTablePO, @Param("oldTableMeta") TablePO oldTablePO) { + return getProvider().updateTableMeta(newTablePO, oldTablePO); + } + + public static String softDeleteTableMetasByTableId(@Param("tableId") Long tableId) { + return getProvider().softDeleteTableMetasByTableId(tableId); + } + + public static String softDeleteTableMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTableMetasByMetalakeId(metalakeId); + } + + public static String softDeleteTableMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteTableMetasByCatalogId(catalogId); + } + + public static String softDeleteTableMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteTableMetasBySchemaId(schemaId); + } + + public static String deleteTableMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTableMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaBaseSQLProvider.java new file mode 100644 index 00000000000..4c9ee54bfff --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaBaseSQLProvider.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TagMetaMapper.TAG_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.TagPO; +import org.apache.ibatis.annotations.Param; + +public class TagMetaBaseSQLProvider { + + public String listTagPOsByMetalake(@Param("metalakeName") String metalakeName) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId," + + " tm.tag_comment as comment," + + " tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String listTagPOsByMetalakeAndTagNames( + @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames) { + return ""; + } + + public String selectTagIdByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "SELECT tm.tag_id as tagId FROM " + + TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String selectTagMetaByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId," + + " tm.tag_comment as comment," + + " tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String insertTagMeta(@Param("tagMeta") TagPO tagPO) { + return "INSERT INTO " + + TAG_TABLE_NAME + + " (tag_id, tag_name," + + " metalake_id, tag_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tagMeta.tagId}," + + " #{tagMeta.tagName}," + + " #{tagMeta.metalakeId}," + + " #{tagMeta.comment}," + + " #{tagMeta.properties}," + + " #{tagMeta.auditInfo}," + + " #{tagMeta.currentVersion}," + + " #{tagMeta.lastVersion}," + + " #{tagMeta.deletedAt}" + + " )"; + } + + public String insertTagMetaOnDuplicateKeyUpdate(@Param("tagMeta") TagPO tagPO) { + return "INSERT INTO " + + TAG_TABLE_NAME + + "(tag_id, tag_name," + + " metalake_id, tag_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tagMeta.tagId}," + + " #{tagMeta.tagName}," + + " #{tagMeta.metalakeId}," + + " #{tagMeta.comment}," + + " #{tagMeta.properties}," + + " #{tagMeta.auditInfo}," + + " #{tagMeta.currentVersion}," + + " #{tagMeta.lastVersion}," + + " #{tagMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " tag_name = #{tagMeta.tagName}," + + " metalake_id = #{tagMeta.metalakeId}," + + " tag_comment = #{tagMeta.comment}," + + " properties = #{tagMeta.properties}," + + " audit_info = #{tagMeta.auditInfo}," + + " current_version = #{tagMeta.currentVersion}," + + " last_version = #{tagMeta.lastVersion}," + + " deleted_at = #{tagMeta.deletedAt}"; + } + + public String updateTagMeta( + @Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET tag_name = #{newTagMeta.tagName}," + + " tag_comment = #{newTagMeta.comment}," + + " properties = #{newTagMeta.properties}," + + " audit_info = #{newTagMeta.auditInfo}," + + " current_version = #{newTagMeta.currentVersion}," + + " last_version = #{newTagMeta.lastVersion}," + + " deleted_at = #{newTagMeta.deletedAt}" + + " WHERE tag_id = #{oldTagMeta.tagId}" + + " AND metalake_id = #{oldTagMeta.metalakeId}" + + " AND tag_name = #{oldTagMeta.tagName}" + + " AND (tag_comment IS NULL OR tag_comment = #{oldTagMeta.comment})" + + " AND properties = #{oldTagMeta.properties}" + + " AND audit_info = #{oldTagMeta.auditInfo}" + + " AND current_version = #{oldTagMeta.currentVersion}" + + " AND last_version = #{oldTagMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String softDeleteTagMetaByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "UPDATE " + + TAG_TABLE_NAME + + " tm SET tm.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE tm.metalake_id IN (" + + " SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0"; + } + + public String softDeleteTagMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String deleteTagMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TAG_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java index ad8afcfa366..a76a459f42d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -20,181 +20,54 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TagPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; public interface TagMetaMapper { String TAG_TABLE_NAME = "tag_meta"; - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId," - + " tm.tag_comment as comment," - + " tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider(type = TagMetaSQLProviderFactory.class, method = "listTagPOsByMetalake") List listTagPOsByMetalake(@Param("metalakeName") String metalakeName); - @Select( - "") + @SelectProvider( + type = TagMetaSQLProviderFactory.class, + method = "listTagPOsByMetalakeAndTagNames") List listTagPOsByMetalakeAndTagNames( @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames); - @Select( - "SELECT tm.tag_id as tagId FROM " - + TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" - + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider(type = TagMetaSQLProviderFactory.class, method = "selectTagIdByMetalakeAndName") Long selectTagIdByMetalakeAndName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId," - + " tm.tag_comment as comment," - + " tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" - + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider(type = TagMetaSQLProviderFactory.class, method = "selectTagMetaByMetalakeAndName") TagPO selectTagMetaByMetalakeAndName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Insert( - "INSERT INTO " - + TAG_TABLE_NAME - + " (tag_id, tag_name," - + " metalake_id, tag_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tagMeta.tagId}," - + " #{tagMeta.tagName}," - + " #{tagMeta.metalakeId}," - + " #{tagMeta.comment}," - + " #{tagMeta.properties}," - + " #{tagMeta.auditInfo}," - + " #{tagMeta.currentVersion}," - + " #{tagMeta.lastVersion}," - + " #{tagMeta.deletedAt}" - + " )") + @InsertProvider(type = TagMetaSQLProviderFactory.class, method = "insertTagMeta") void insertTagMeta(@Param("tagMeta") TagPO tagPO); - @Insert( - "INSERT INTO " - + TAG_TABLE_NAME - + "(tag_id, tag_name," - + " metalake_id, tag_comment, properties, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{tagMeta.tagId}," - + " #{tagMeta.tagName}," - + " #{tagMeta.metalakeId}," - + " #{tagMeta.comment}," - + " #{tagMeta.properties}," - + " #{tagMeta.auditInfo}," - + " #{tagMeta.currentVersion}," - + " #{tagMeta.lastVersion}," - + " #{tagMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " tag_name = #{tagMeta.tagName}," - + " metalake_id = #{tagMeta.metalakeId}," - + " tag_comment = #{tagMeta.comment}," - + " properties = #{tagMeta.properties}," - + " audit_info = #{tagMeta.auditInfo}," - + " current_version = #{tagMeta.currentVersion}," - + " last_version = #{tagMeta.lastVersion}," - + " deleted_at = #{tagMeta.deletedAt}") + @InsertProvider( + type = TagMetaSQLProviderFactory.class, + method = "insertTagMetaOnDuplicateKeyUpdate") void insertTagMetaOnDuplicateKeyUpdate(@Param("tagMeta") TagPO tagPO); - @Update( - "UPDATE " - + TAG_TABLE_NAME - + " SET tag_name = #{newTagMeta.tagName}," - + " tag_comment = #{newTagMeta.comment}," - + " properties = #{newTagMeta.properties}," - + " audit_info = #{newTagMeta.auditInfo}," - + " current_version = #{newTagMeta.currentVersion}," - + " last_version = #{newTagMeta.lastVersion}," - + " deleted_at = #{newTagMeta.deletedAt}" - + " WHERE tag_id = #{oldTagMeta.tagId}" - + " AND metalake_id = #{oldTagMeta.metalakeId}" - + " AND tag_name = #{oldTagMeta.tagName}" - + " AND (tag_comment IS NULL OR tag_comment = #{oldTagMeta.comment})" - + " AND properties = #{oldTagMeta.properties}" - + " AND audit_info = #{oldTagMeta.auditInfo}" - + " AND current_version = #{oldTagMeta.currentVersion}" - + " AND last_version = #{oldTagMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = TagMetaSQLProviderFactory.class, method = "updateTagMeta") Integer updateTagMeta(@Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO); - @Update( - "UPDATE " - + TAG_TABLE_NAME - + " tm SET tm.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE tm.metalake_id IN (" - + " SELECT mm.metalake_id FROM " - + MetalakeMetaMapper.TABLE_NAME - + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0") + @UpdateProvider( + type = TagMetaSQLProviderFactory.class, + method = "softDeleteTagMetaByMetalakeAndTagName") Integer softDeleteTagMetaByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Update( - "UPDATE " - + TAG_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider(type = TagMetaSQLProviderFactory.class, method = "softDeleteTagMetasByMetalakeId") void softDeleteTagMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TAG_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider(type = TagMetaSQLProviderFactory.class, method = "deleteTagMetasByLegacyTimeline") Integer deleteTagMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java new file mode 100644 index 00000000000..c9a018c5f2c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaSQLProviderFactory.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TagMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.TagPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TagMetaSQLProviderFactory { + + private static final Map METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TagMetaMySQLProvider(), + JDBCBackendType.H2, new TagMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new TagMetaPostgreSQLProvider()); + + public static TagMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TagMetaMySQLProvider extends TagMetaBaseSQLProvider {} + + static class TagMetaH2Provider extends TagMetaBaseSQLProvider {} + + public static String listTagPOsByMetalake(@Param("metalakeName") String metalakeName) { + return getProvider().listTagPOsByMetalake(metalakeName); + } + + public static String listTagPOsByMetalakeAndTagNames( + @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames) { + return getProvider().listTagPOsByMetalakeAndTagNames(metalakeName, tagNames); + } + + public static String selectTagIdByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().selectTagIdByMetalakeAndName(metalakeName, tagName); + } + + public static String selectTagMetaByMetalakeAndName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().selectTagMetaByMetalakeAndName(metalakeName, tagName); + } + + public static String insertTagMeta(@Param("tagMeta") TagPO tagPO) { + return getProvider().insertTagMeta(tagPO); + } + + public static String insertTagMetaOnDuplicateKeyUpdate(@Param("tagMeta") TagPO tagPO) { + return getProvider().insertTagMetaOnDuplicateKeyUpdate(tagPO); + } + + public static String updateTagMeta( + @Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO) { + return getProvider().updateTagMeta(newTagPO, oldTagPO); + } + + public static String softDeleteTagMetaByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().softDeleteTagMetaByMetalakeAndTagName(metalakeName, tagName); + } + + public static String softDeleteTagMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTagMetasByMetalakeId(metalakeId); + } + + public static String deleteTagMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTagMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelBaseSQLProvider.java new file mode 100644 index 00000000000..9bcea1cd95e --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelBaseSQLProvider.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; +import org.apache.ibatis.annotations.Param; + +public class TagMetadataObjectRelBaseSQLProvider { + + public String listTagPOsByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND te.deleted_at = 0" + + " AND tm.deleted_at = 0"; + } + + public String getTagPOsByMetadataObjectAndTagName( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagName") String tagName) { + return "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0"; + } + + public String listTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," + + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," + + " te.current_version as currentVersion, te.last_version as lastVersion," + + " te.deleted_at as deletedAt" + + " FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te JOIN " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON te.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } + + public String batchInsertTagMetadataObjectRels( + @Param("tagRels") List tagRelPOs) { + return ""; + } + + public String batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagIds") List tagIds) { + return ""; + } + + public String softDeleteTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + public String softDeleteTagMetadataObjectRelsByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE EXISTS (SELECT * FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + public String deleteTagEntityRelsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index 1ade27d8ffc..40b62955881 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -21,139 +21,63 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.po.TagPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; public interface TagMetadataObjectRelMapper { String TAG_METADATA_OBJECT_RELATION_TABLE_NAME = "tag_relation_meta"; - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm JOIN " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te ON tm.tag_id = te.tag_id" - + " WHERE te.metadata_object_id = #{metadataObjectId}" - + " AND te.metadata_object_type = #{metadataObjectType} AND te.deleted_at = 0" - + " AND tm.deleted_at = 0") + @SelectProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "listTagPOsByMetadataObjectIdAndType") List listTagPOsByMetadataObjectIdAndType( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType); - @Select( - "SELECT tm.tag_id as tagId, tm.tag_name as tagName," - + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," - + " tm.audit_info as auditInfo," - + " tm.current_version as currentVersion," - + " tm.last_version as lastVersion," - + " tm.deleted_at as deletedAt" - + " FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm JOIN " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te ON tm.tag_id = te.tag_id" - + " WHERE te.metadata_object_id = #{metadataObjectId}" - + " AND te.metadata_object_type = #{metadataObjectType} AND tm.tag_name = #{tagName}" - + " AND te.deleted_at = 0 AND tm.deleted_at = 0") + @SelectProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "getTagPOsByMetadataObjectAndTagName") TagPO getTagPOsByMetadataObjectAndTagName( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType, @Param("tagName") String tagName); - @Select( - "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," - + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," - + " te.current_version as currentVersion, te.last_version as lastVersion," - + " te.deleted_at as deletedAt" - + " FROM " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te JOIN " - + TagMetaMapper.TAG_TABLE_NAME - + " tm JOIN " - + MetalakeMetaMapper.TABLE_NAME - + " mm ON te.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" - + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0") + @SelectProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "listTagMetadataObjectRelsByMetalakeAndTagName") List listTagMetadataObjectRelsByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Insert({ - "" - }) + @InsertProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "batchInsertTagMetadataObjectRels") void batchInsertTagMetadataObjectRels(@Param("tagRels") List tagRelPOs); - @Update({ - "" - }) + @UpdateProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject") void batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( @Param("metadataObjectId") Long metadataObjectId, @Param("metadataObjectType") String metadataObjectType, @Param("tagIds") List tagIds); - @Update( - "UPDATE " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " - + MetalakeMetaMapper.TABLE_NAME - + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.deleted_at = 0) AND te.deleted_at = 0") + @UpdateProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "softDeleteTagMetadataObjectRelsByMetalakeAndTagName") Integer softDeleteTagMetadataObjectRelsByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); - @Update( - "UPDATE " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE EXISTS (SELECT * FROM " - + TagMetaMapper.TAG_TABLE_NAME - + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" - + " AND tm.deleted_at = 0) AND te.deleted_at = 0") + @UpdateProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "softDeleteTagMetadataObjectRelsByMetalakeId") void softDeleteTagMetadataObjectRelsByMetalakeId(@Param("metalakeId") Long metalakeId); - @Delete( - "DELETE FROM " - + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = TagMetadataObjectRelSQLProviderFactory.class, + method = "deleteTagEntityRelsByLegacyTimeline") Integer deleteTagEntityRelsByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java new file mode 100644 index 00000000000..af4522f9d95 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TagMetadataObjectRelPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TagMetadataObjectRelSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TagMetadataObjectRelMySQLProvider(), + JDBCBackendType.H2, new TagMetadataObjectRelH2Provider(), + JDBCBackendType.POSTGRESQL, new TagMetadataObjectRelPostgreSQLProvider()); + + public static TagMetadataObjectRelBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TagMetadataObjectRelMySQLProvider extends TagMetadataObjectRelBaseSQLProvider {} + + static class TagMetadataObjectRelH2Provider extends TagMetadataObjectRelBaseSQLProvider {} + + public static String listTagPOsByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType) { + return getProvider().listTagPOsByMetadataObjectIdAndType(metadataObjectId, metadataObjectType); + } + + public static String getTagPOsByMetadataObjectAndTagName( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagName") String tagName) { + return getProvider() + .getTagPOsByMetadataObjectAndTagName(metadataObjectId, metadataObjectType, tagName); + } + + public static String listTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().listTagMetadataObjectRelsByMetalakeAndTagName(metalakeName, tagName); + } + + public static String batchInsertTagMetadataObjectRels( + @Param("tagRels") List tagRelPOs) { + return getProvider().batchInsertTagMetadataObjectRels(tagRelPOs); + } + + public static String batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagIds") List tagIds) { + return getProvider() + .batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + metadataObjectId, metadataObjectType, tagIds); + } + + public static String softDeleteTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName) { + return getProvider().softDeleteTagMetadataObjectRelsByMetalakeAndTagName(metalakeName, tagName); + } + + public static String softDeleteTagMetadataObjectRelsByMetalakeId( + @Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTagMetadataObjectRelsByMetalakeId(metalakeId); + } + + public static String deleteTagEntityRelsByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTagEntityRelsByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaBaseSQLProvider.java new file mode 100644 index 00000000000..aa46bbd7f9c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaBaseSQLProvider.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.TopicPO; +import org.apache.ibatis.annotations.Param; + +public class TopicMetaBaseSQLProvider { + + public String insertTopicMeta(@Param("topicMeta") TopicPO topicPO) { + return "INSERT INTO " + + TABLE_NAME + + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," + + " comment, properties, audit_info, current_version, last_version," + + " deleted_at)" + + " VALUES(" + + " #{topicMeta.topicId}," + + " #{topicMeta.topicName}," + + " #{topicMeta.metalakeId}," + + " #{topicMeta.catalogId}," + + " #{topicMeta.schemaId}," + + " #{topicMeta.comment}," + + " #{topicMeta.properties}," + + " #{topicMeta.auditInfo}," + + " #{topicMeta.currentVersion}," + + " #{topicMeta.lastVersion}," + + " #{topicMeta.deletedAt}" + + " )"; + } + + public String insertTopicMetaOnDuplicateKeyUpdate(@Param("topicMeta") TopicPO topicPO) { + return "INSERT INTO " + + TABLE_NAME + + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," + + " comment, properties, audit_info, current_version, last_version," + + " deleted_at)" + + " VALUES(" + + " #{topicMeta.topicId}," + + " #{topicMeta.topicName}," + + " #{topicMeta.metalakeId}," + + " #{topicMeta.catalogId}," + + " #{topicMeta.schemaId}," + + " #{topicMeta.comment}," + + " #{topicMeta.properties}," + + " #{topicMeta.auditInfo}," + + " #{topicMeta.currentVersion}," + + " #{topicMeta.lastVersion}," + + " #{topicMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " topic_name = #{topicMeta.topicName}," + + " metalake_id = #{topicMeta.metalakeId}," + + " catalog_id = #{topicMeta.catalogId}," + + " schema_id = #{topicMeta.schemaId}," + + " comment = #{topicMeta.comment}," + + " properties = #{topicMeta.properties}," + + " audit_info = #{topicMeta.auditInfo}," + + " current_version = #{topicMeta.currentVersion}," + + " last_version = #{topicMeta.lastVersion}," + + " deleted_at = #{topicMeta.deletedAt}"; + } + + public String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) { + return "SELECT topic_id as topicId, topic_name as topicName, metalake_id as metalakeId," + + " catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String selectTopicMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String topicName) { + return "SELECT topic_id as topicId, topic_name as topicName," + + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName} AND deleted_at = 0"; + } + + public String selectTopicMetaById(@Param("topicId") Long topicId) { + return "SELECT topic_id as topicId, topic_name as topicName," + + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE topic_id = #{topicId} AND deleted_at = 0"; + } + + public String updateTopicMeta( + @Param("newTopicMeta") TopicPO newTopicPO, @Param("oldTopicMeta") TopicPO oldTopicPO) { + return "UPDATE " + + TABLE_NAME + + " SET topic_name = #{newTopicMeta.topicName}," + + " metalake_id = #{newTopicMeta.metalakeId}," + + " catalog_id = #{newTopicMeta.catalogId}," + + " schema_id = #{newTopicMeta.schemaId}," + + " comment = #{newTopicMeta.comment}," + + " properties = #{newTopicMeta.properties}," + + " audit_info = #{newTopicMeta.auditInfo}," + + " current_version = #{newTopicMeta.currentVersion}," + + " last_version = #{newTopicMeta.lastVersion}," + + " deleted_at = #{newTopicMeta.deletedAt}" + + " WHERE topic_id = #{oldTopicMeta.topicId}" + + " AND topic_name = #{oldTopicMeta.topicName}" + + " AND metalake_id = #{oldTopicMeta.metalakeId}" + + " AND catalog_id = #{oldTopicMeta.catalogId}" + + " AND schema_id = #{oldTopicMeta.schemaId}" + + " AND comment = #{oldTopicMeta.comment}" + + " AND properties = #{oldTopicMeta.properties}" + + " AND audit_info = #{oldTopicMeta.auditInfo}" + + " AND current_version = #{oldTopicMeta.currentVersion}" + + " AND last_version = #{oldTopicMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String selectTopicIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String name) { + return "SELECT topic_id as topicId FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName}" + + " AND deleted_at = 0"; + } + + public String softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE topic_id = #{topicId} AND deleted_at = 0"; + } + + public String softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + public String softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + public String deleteTopicMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + TABLE_NAME + + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java index d8d08286e38..8c194caff4f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaMapper.java @@ -20,173 +20,68 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.TopicPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; public interface TopicMetaMapper { String TABLE_NAME = "topic_meta"; - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," - + " comment, properties, audit_info, current_version, last_version," - + " deleted_at)" - + " VALUES(" - + " #{topicMeta.topicId}," - + " #{topicMeta.topicName}," - + " #{topicMeta.metalakeId}," - + " #{topicMeta.catalogId}," - + " #{topicMeta.schemaId}," - + " #{topicMeta.comment}," - + " #{topicMeta.properties}," - + " #{topicMeta.auditInfo}," - + " #{topicMeta.currentVersion}," - + " #{topicMeta.lastVersion}," - + " #{topicMeta.deletedAt}" - + " )") + @InsertProvider(type = TopicMetaSQLProviderFactory.class, method = "insertTopicMeta") void insertTopicMeta(@Param("topicMeta") TopicPO topicPO); - @Insert( - "INSERT INTO " - + TABLE_NAME - + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," - + " comment, properties, audit_info, current_version, last_version," - + " deleted_at)" - + " VALUES(" - + " #{topicMeta.topicId}," - + " #{topicMeta.topicName}," - + " #{topicMeta.metalakeId}," - + " #{topicMeta.catalogId}," - + " #{topicMeta.schemaId}," - + " #{topicMeta.comment}," - + " #{topicMeta.properties}," - + " #{topicMeta.auditInfo}," - + " #{topicMeta.currentVersion}," - + " #{topicMeta.lastVersion}," - + " #{topicMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " topic_name = #{topicMeta.topicName}," - + " metalake_id = #{topicMeta.metalakeId}," - + " catalog_id = #{topicMeta.catalogId}," - + " schema_id = #{topicMeta.schemaId}," - + " comment = #{topicMeta.comment}," - + " properties = #{topicMeta.properties}," - + " audit_info = #{topicMeta.auditInfo}," - + " current_version = #{topicMeta.currentVersion}," - + " last_version = #{topicMeta.lastVersion}," - + " deleted_at = #{topicMeta.deletedAt}") + @InsertProvider( + type = TopicMetaSQLProviderFactory.class, + method = "insertTopicMetaOnDuplicateKeyUpdate") void insertTopicMetaOnDuplicateKeyUpdate(@Param("topicMeta") TopicPO topicPO); - @Select( - "SELECT topic_id as topicId, topic_name as topicName, metalake_id as metalakeId," - + " catalog_id as catalogId, schema_id as schemaId," - + " comment as comment, properties as properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsBySchemaId") List listTopicPOsBySchemaId(@Param("schemaId") Long schemaId); - @Select( - "SELECT topic_id as topicId, topic_name as topicName," - + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," - + " comment as comment, properties as properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName} AND deleted_at = 0") + @SelectProvider( + type = TopicMetaSQLProviderFactory.class, + method = "selectTopicMetaBySchemaIdAndName") TopicPO selectTopicMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String topicName); - @Select( - "SELECT topic_id as topicId, topic_name as topicName," - + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," - + " comment as comment, properties as properties, audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + TABLE_NAME - + " WHERE topic_id = #{topicId} AND deleted_at = 0") + @SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "selectTopicMetaById") TopicPO selectTopicMetaById(@Param("topicId") Long topicId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET topic_name = #{newTopicMeta.topicName}," - + " metalake_id = #{newTopicMeta.metalakeId}," - + " catalog_id = #{newTopicMeta.catalogId}," - + " schema_id = #{newTopicMeta.schemaId}," - + " comment = #{newTopicMeta.comment}," - + " properties = #{newTopicMeta.properties}," - + " audit_info = #{newTopicMeta.auditInfo}," - + " current_version = #{newTopicMeta.currentVersion}," - + " last_version = #{newTopicMeta.lastVersion}," - + " deleted_at = #{newTopicMeta.deletedAt}" - + " WHERE topic_id = #{oldTopicMeta.topicId}" - + " AND topic_name = #{oldTopicMeta.topicName}" - + " AND metalake_id = #{oldTopicMeta.metalakeId}" - + " AND catalog_id = #{oldTopicMeta.catalogId}" - + " AND schema_id = #{oldTopicMeta.schemaId}" - + " AND comment = #{oldTopicMeta.comment}" - + " AND properties = #{oldTopicMeta.properties}" - + " AND audit_info = #{oldTopicMeta.auditInfo}" - + " AND current_version = #{oldTopicMeta.currentVersion}" - + " AND last_version = #{oldTopicMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = TopicMetaSQLProviderFactory.class, method = "updateTopicMeta") Integer updateTopicMeta( @Param("newTopicMeta") TopicPO newTopicPO, @Param("oldTopicMeta") TopicPO oldTopicPO); - @Select( - "SELECT topic_id as topicId FROM " - + TABLE_NAME - + " WHERE schema_id = #{schemaId} AND topic_name = #{topicName}" - + " AND deleted_at = 0") + @SelectProvider( + type = TopicMetaSQLProviderFactory.class, + method = "selectTopicIdBySchemaIdAndName") Long selectTopicIdBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String name); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE topic_id = #{topicId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasByTopicId") Integer softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasByCatalogId") Integer softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasByMetalakeId") Integer softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + @UpdateProvider( + type = TopicMetaSQLProviderFactory.class, + method = "softDeleteTopicMetasBySchemaId") Integer softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId); - @Delete( - "DELETE FROM " - + TABLE_NAME - + " WHERE deleted_at != 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = TopicMetaSQLProviderFactory.class, + method = "deleteTopicMetasByLegacyTimeline") Integer deleteTopicMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java new file mode 100644 index 00000000000..c1c6e2e237d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TopicMetaSQLProviderFactory.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.TopicMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.TopicPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class TopicMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new TopicMetaMySQLProvider(), + JDBCBackendType.H2, new TopicMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new TopicMetaPostgreSQLProvider()); + + public static TopicMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class TopicMetaMySQLProvider extends TopicMetaBaseSQLProvider {} + + static class TopicMetaH2Provider extends TopicMetaBaseSQLProvider {} + + public static String insertTopicMeta(@Param("topicMeta") TopicPO topicPO) { + return getProvider().insertTopicMeta(topicPO); + } + + public static String insertTopicMetaOnDuplicateKeyUpdate(@Param("topicMeta") TopicPO topicPO) { + return getProvider().insertTopicMetaOnDuplicateKeyUpdate(topicPO); + } + + public static String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().listTopicPOsBySchemaId(schemaId); + } + + public static String selectTopicMetaBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String topicName) { + return getProvider().selectTopicMetaBySchemaIdAndName(schemaId, topicName); + } + + public static String selectTopicMetaById(@Param("topicId") Long topicId) { + return getProvider().selectTopicMetaById(topicId); + } + + public static String updateTopicMeta( + @Param("newTopicMeta") TopicPO newTopicPO, @Param("oldTopicMeta") TopicPO oldTopicPO) { + return getProvider().updateTopicMeta(newTopicPO, oldTopicPO); + } + + public static String selectTopicIdBySchemaIdAndName( + @Param("schemaId") Long schemaId, @Param("topicName") String name) { + return getProvider().selectTopicIdBySchemaIdAndName(schemaId, name); + } + + public static String softDeleteTopicMetasByTopicId(@Param("topicId") Long topicId) { + return getProvider().softDeleteTopicMetasByTopicId(topicId); + } + + public static String softDeleteTopicMetasByCatalogId(@Param("catalogId") Long catalogId) { + return getProvider().softDeleteTopicMetasByCatalogId(catalogId); + } + + public static String softDeleteTopicMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteTopicMetasByMetalakeId(metalakeId); + } + + public static String softDeleteTopicMetasBySchemaId(@Param("schemaId") Long schemaId) { + return getProvider().softDeleteTopicMetasBySchemaId(schemaId); + } + + public static String deleteTopicMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteTopicMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaBaseSQLProvider.java new file mode 100644 index 00000000000..a5db8e0f943 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaBaseSQLProvider.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.UserMetaMapper.USER_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.po.UserPO; +import org.apache.ibatis.annotations.Param; + +public class UserMetaBaseSQLProvider { + + public String selectUserIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name) { + return "SELECT user_id as userId FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" + + " AND deleted_at = 0"; + } + + public String selectUserMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name) { + return "SELECT user_id as userId, user_name as userName," + + " metalake_id as metalakeId," + + " audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" + + " AND deleted_at = 0"; + } + + public String insertUserMeta(@Param("userMeta") UserPO userPO) { + return "INSERT INTO " + + USER_TABLE_NAME + + "(user_id, user_name," + + " metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )"; + } + + public String insertUserMetaOnDuplicateKeyUpdate(@Param("userMeta") UserPO userPO) { + return "INSERT INTO " + + USER_TABLE_NAME + + "(user_id, user_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )" + + " ON DUPLICATE KEY UPDATE" + + " user_name = #{userMeta.userName}," + + " metalake_id = #{userMeta.metalakeId}," + + " audit_info = #{userMeta.auditInfo}," + + " current_version = #{userMeta.currentVersion}," + + " last_version = #{userMeta.lastVersion}," + + " deleted_at = #{userMeta.deletedAt}"; + } + + public String softDeleteUserMetaByUserId(@Param("userId") Long userId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + public String softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + public String updateUserMeta( + @Param("newUserMeta") UserPO newUserPO, @Param("oldUserMeta") UserPO oldUserPO) { + return "UPDATE " + + USER_TABLE_NAME + + " SET user_name = #{newUserMeta.userName}," + + " metalake_id = #{newUserMeta.metalakeId}," + + " audit_info = #{newUserMeta.auditInfo}," + + " current_version = #{newUserMeta.currentVersion}," + + " last_version = #{newUserMeta.lastVersion}," + + " deleted_at = #{newUserMeta.deletedAt}" + + " WHERE user_id = #{oldUserMeta.userId}" + + " AND user_name = #{oldUserMeta.userName}" + + " AND metalake_id = #{oldUserMeta.metalakeId}" + + " AND audit_info = #{oldUserMeta.auditInfo}" + + " AND current_version = #{oldUserMeta.currentVersion}" + + " AND last_version = #{oldUserMeta.lastVersion}" + + " AND deleted_at = 0"; + } + + public String listUsersByRoleId(@Param("roleId") Long roleId) { + return "SELECT us.user_id as userId, us.user_name as userName," + + " us.metalake_id as metalakeId," + + " us.audit_info as auditInfo, us.current_version as currentVersion," + + " us.last_version as lastVersion, us.deleted_at as deletedAt" + + " FROM " + + USER_TABLE_NAME + + " us JOIN " + + USER_ROLE_RELATION_TABLE_NAME + + " re ON us.user_id = re.user_id" + + " WHERE re.role_id = #{roleId}" + + " AND us.deleted_at = 0 AND re.deleted_at = 0"; + } + + public String deleteUserMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + USER_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java index e7b442c099a..ad794c39530 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaMapper.java @@ -21,11 +21,11 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.UserPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -39,121 +39,44 @@ public interface UserMetaMapper { String USER_TABLE_NAME = "user_meta"; String USER_ROLE_RELATION_TABLE_NAME = "user_role_rel"; - @Select( - "SELECT user_id as userId FROM " - + USER_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" - + " AND deleted_at = 0") + @SelectProvider( + type = UserMetaSQLProviderFactory.class, + method = "selectUserIdByMetalakeIdAndName") Long selectUserIdByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("userName") String name); - @Select( - "SELECT user_id as userId, user_name as userName," - + " metalake_id as metalakeId," - + " audit_info as auditInfo," - + " current_version as currentVersion, last_version as lastVersion," - + " deleted_at as deletedAt" - + " FROM " - + USER_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND user_name = #{userName}" - + " AND deleted_at = 0") + @SelectProvider( + type = UserMetaSQLProviderFactory.class, + method = "selectUserMetaByMetalakeIdAndName") UserPO selectUserMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("userName") String name); - @Insert( - "INSERT INTO " - + USER_TABLE_NAME - + "(user_id, user_name," - + " metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{userMeta.userId}," - + " #{userMeta.userName}," - + " #{userMeta.metalakeId}," - + " #{userMeta.auditInfo}," - + " #{userMeta.currentVersion}," - + " #{userMeta.lastVersion}," - + " #{userMeta.deletedAt}" - + " )") + @InsertProvider(type = UserMetaSQLProviderFactory.class, method = "insertUserMeta") void insertUserMeta(@Param("userMeta") UserPO userPO); - @Insert( - "INSERT INTO " - + USER_TABLE_NAME - + "(user_id, user_name," - + "metalake_id, audit_info," - + " current_version, last_version, deleted_at)" - + " VALUES(" - + " #{userMeta.userId}," - + " #{userMeta.userName}," - + " #{userMeta.metalakeId}," - + " #{userMeta.auditInfo}," - + " #{userMeta.currentVersion}," - + " #{userMeta.lastVersion}," - + " #{userMeta.deletedAt}" - + " )" - + " ON DUPLICATE KEY UPDATE" - + " user_name = #{userMeta.userName}," - + " metalake_id = #{userMeta.metalakeId}," - + " audit_info = #{userMeta.auditInfo}," - + " current_version = #{userMeta.currentVersion}," - + " last_version = #{userMeta.lastVersion}," - + " deleted_at = #{userMeta.deletedAt}") + @InsertProvider( + type = UserMetaSQLProviderFactory.class, + method = "insertUserMetaOnDuplicateKeyUpdate") void insertUserMetaOnDuplicateKeyUpdate(@Param("userMeta") UserPO userPO); - @Update( - "UPDATE " - + USER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE user_id = #{userId} AND deleted_at = 0") + @UpdateProvider(type = UserMetaSQLProviderFactory.class, method = "softDeleteUserMetaByUserId") void softDeleteUserMetaByUserId(@Param("userId") Long userId); - @Update( - "UPDATE " - + USER_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" - + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0") + @UpdateProvider( + type = UserMetaSQLProviderFactory.class, + method = "softDeleteUserMetasByMetalakeId") void softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + USER_TABLE_NAME - + " SET user_name = #{newUserMeta.userName}," - + " metalake_id = #{newUserMeta.metalakeId}," - + " audit_info = #{newUserMeta.auditInfo}," - + " current_version = #{newUserMeta.currentVersion}," - + " last_version = #{newUserMeta.lastVersion}," - + " deleted_at = #{newUserMeta.deletedAt}" - + " WHERE user_id = #{oldUserMeta.userId}" - + " AND user_name = #{oldUserMeta.userName}" - + " AND metalake_id = #{oldUserMeta.metalakeId}" - + " AND audit_info = #{oldUserMeta.auditInfo}" - + " AND current_version = #{oldUserMeta.currentVersion}" - + " AND last_version = #{oldUserMeta.lastVersion}" - + " AND deleted_at = 0") + @UpdateProvider(type = UserMetaSQLProviderFactory.class, method = "updateUserMeta") Integer updateUserMeta( @Param("newUserMeta") UserPO newUserPO, @Param("oldUserMeta") UserPO oldUserPO); - @Select( - "SELECT us.user_id as userId, us.user_name as userName," - + " us.metalake_id as metalakeId," - + " us.audit_info as auditInfo, us.current_version as currentVersion," - + " us.last_version as lastVersion, us.deleted_at as deletedAt" - + " FROM " - + USER_TABLE_NAME - + " us JOIN " - + USER_ROLE_RELATION_TABLE_NAME - + " re ON us.user_id = re.user_id" - + " WHERE re.role_id = #{roleId}" - + " AND us.deleted_at = 0 AND re.deleted_at = 0") + @SelectProvider(type = UserMetaSQLProviderFactory.class, method = "listUsersByRoleId") List listUsersByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + USER_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = UserMetaSQLProviderFactory.class, + method = "deleteUserMetasByLegacyTimeline") Integer deleteUserMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java new file mode 100644 index 00000000000..2c322db86d6 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserMetaSQLProviderFactory.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.UserMetaPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.UserPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class UserMetaSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new UserMetaMySQLProvider(), + JDBCBackendType.H2, new UserMetaH2Provider(), + JDBCBackendType.POSTGRESQL, new UserMetaPostgreSQLProvider()); + + public static UserMetaBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class UserMetaMySQLProvider extends UserMetaBaseSQLProvider {} + + static class UserMetaH2Provider extends UserMetaBaseSQLProvider {} + + public static String selectUserIdByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String userName) { + return getProvider().selectUserIdByMetalakeIdAndName(metalakeId, userName); + } + + public static String selectUserMetaByMetalakeIdAndName( + @Param("metalakeId") Long metalakeId, @Param("userName") String name) { + return getProvider().selectUserMetaByMetalakeIdAndName(metalakeId, name); + } + + public static String insertUserMeta(@Param("userMeta") UserPO userPO) { + return getProvider().insertUserMeta(userPO); + } + + public static String insertUserMetaOnDuplicateKeyUpdate(@Param("userMeta") UserPO userPO) { + return getProvider().insertUserMetaOnDuplicateKeyUpdate(userPO); + } + + public static String softDeleteUserMetaByUserId(@Param("userId") Long userId) { + return getProvider().softDeleteUserMetaByUserId(userId); + } + + public static String softDeleteUserMetasByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteUserMetasByMetalakeId(metalakeId); + } + + public static String updateUserMeta( + @Param("newUserMeta") UserPO newUserPO, @Param("oldUserMeta") UserPO oldUserPO) { + return getProvider().updateUserMeta(newUserPO, oldUserPO); + } + + public static String listUsersByRoleId(@Param("roleId") Long roleId) { + return getProvider().listUsersByRoleId(roleId); + } + + public static String deleteUserMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteUserMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelBaseSQLProvider.java new file mode 100644 index 00000000000..a1e6659182d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelBaseSQLProvider.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.po.UserRoleRelPO; +import org.apache.ibatis.annotations.Param; + +public class UserRoleRelBaseSQLProvider { + + public String batchInsertUserRoleRel( + @Param("userRoleRelList") List userRoleRelList) { + return ""; + } + + public String batchInsertUserRoleRelOnDuplicateKeyUpdate( + @Param("userRoleRels") List userRoleRelPOs) { + return ""; + } + + public String softDeleteUserRoleRelByUserId(@Param("userId") Long userId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " + + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + public String softDeleteUserRoleRelByUserAndRoles( + @Param("userId") Long userId, @Param("roleIds") List roleIds) { + return ""; + } + + public String softDeleteUserRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " + + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE user_id IN (SELECT user_id FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + public String softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " + + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + public String deleteUserRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return "DELETE FROM " + + USER_ROLE_RELATION_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java index 23c9d6702cb..5e540de8fe2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelMapper.java @@ -21,10 +21,10 @@ import java.util.List; import org.apache.gravitino.storage.relational.po.UserRoleRelPO; -import org.apache.ibatis.annotations.Delete; -import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.DeleteProvider; +import org.apache.ibatis.annotations.InsertProvider; import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.UpdateProvider; /** * A MyBatis Mapper for table meta operation SQLs. @@ -38,101 +38,39 @@ public interface UserRoleRelMapper { String USER_TABLE_NAME = "user_meta"; String USER_ROLE_RELATION_TABLE_NAME = "user_role_rel"; - @Insert({ - "" - }) + @InsertProvider(type = UserRoleRelSQLProviderFactory.class, method = "batchInsertUserRoleRel") void batchInsertUserRoleRel(@Param("userRoleRels") List userRoleRelPOs); - @Insert({ - "" - }) + @InsertProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "batchInsertUserRoleRelOnDuplicateKeyUpdate") void batchInsertUserRoleRelOnDuplicateKeyUpdate( @Param("userRoleRels") List userRoleRelPOs); - @Update( - "UPDATE " - + USER_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " - + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE user_id = #{userId} AND deleted_at = 0") + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByUserId") void softDeleteUserRoleRelByUserId(@Param("userId") Long userId); - @Update({ - "" - }) + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByUserAndRoles") void softDeleteUserRoleRelByUserAndRoles( @Param("userId") Long userId, @Param("roleIds") List roleIds); - @Update( - "UPDATE " - + USER_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " - + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE user_id IN (SELECT user_id FROM " - + USER_TABLE_NAME - + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" - + " AND deleted_at = 0") - void softDeleteUserRoleRelByMetalakeId(Long metalakeId); + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByMetalakeId") + void softDeleteUserRoleRelByMetalakeId(@Param("metalakeId") Long metalakeId); - @Update( - "UPDATE " - + USER_ROLE_RELATION_TABLE_NAME - + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0) " - + "+ EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE role_id = #{roleId} AND deleted_at = 0") + @UpdateProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "softDeleteUserRoleRelByRoleId") void softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId); - @Delete( - "DELETE FROM " - + USER_ROLE_RELATION_TABLE_NAME - + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}") + @DeleteProvider( + type = UserRoleRelSQLProviderFactory.class, + method = "deleteUserRoleRelMetasByLegacyTimeline") Integer deleteUserRoleRelMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java new file mode 100644 index 00000000000..f98f509f8f2 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/UserRoleRelSQLProviderFactory.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; +import org.apache.gravitino.storage.relational.mapper.postgresql.UserRoleRelPostgreSQLProvider; +import org.apache.gravitino.storage.relational.po.UserRoleRelPO; +import org.apache.gravitino.storage.relational.session.SqlSessionFactoryHelper; +import org.apache.ibatis.annotations.Param; + +public class UserRoleRelSQLProviderFactory { + + private static final Map + METALAKE_META_SQL_PROVIDER_MAP = + ImmutableMap.of( + JDBCBackendType.MYSQL, new UserRoleRelMySQLProvider(), + JDBCBackendType.H2, new UserRoleRelH2Provider(), + JDBCBackendType.POSTGRESQL, new UserRoleRelPostgreSQLProvider()); + + public static UserRoleRelBaseSQLProvider getProvider() { + String databaseId = + SqlSessionFactoryHelper.getInstance() + .getSqlSessionFactory() + .getConfiguration() + .getDatabaseId(); + + JDBCBackendType jdbcBackendType = JDBCBackendType.fromString(databaseId); + return METALAKE_META_SQL_PROVIDER_MAP.get(jdbcBackendType); + } + + static class UserRoleRelMySQLProvider extends UserRoleRelBaseSQLProvider {} + + static class UserRoleRelH2Provider extends UserRoleRelBaseSQLProvider {} + + public static String batchInsertUserRoleRel( + @Param("userRoleRels") List userRoleRelPOs) { + return getProvider().batchInsertUserRoleRel(userRoleRelPOs); + } + + public static String batchInsertUserRoleRelOnDuplicateKeyUpdate( + @Param("userRoleRels") List userRoleRelPOs) { + return getProvider().batchInsertUserRoleRelOnDuplicateKeyUpdate(userRoleRelPOs); + } + + public static String softDeleteUserRoleRelByUserId(@Param("userId") Long userId) { + return getProvider().softDeleteUserRoleRelByUserId(userId); + } + + public static String softDeleteUserRoleRelByUserAndRoles( + @Param("userId") Long userId, @Param("roleIds") List roleIds) { + return getProvider().softDeleteUserRoleRelByUserAndRoles(userId, roleIds); + } + + public static String softDeleteUserRoleRelByMetalakeId(@Param("metalakeId") Long metalakeId) { + return getProvider().softDeleteUserRoleRelByMetalakeId(metalakeId); + } + + public static String softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId) { + return getProvider().softDeleteUserRoleRelByRoleId(roleId); + } + + public static String deleteUserRoleRelMetasByLegacyTimeline( + @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { + return getProvider().deleteUserRoleRelMetasByLegacyTimeline(legacyTimeline, limit); + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/CatalogMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/CatalogMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..a4646fde852 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/CatalogMetaPostgreSQLProvider.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.CatalogMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.CatalogPO; +import org.apache.ibatis.annotations.Param; + +public class CatalogMetaPostgreSQLProvider extends CatalogMetaBaseSQLProvider { + @Override + public String softDeleteCatalogMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteCatalogMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertCatalogMetaOnDuplicateKeyUpdate(CatalogPO catalogPO) { + return "INSERT INTO " + + TABLE_NAME + + "(catalog_id, catalog_name, metalake_id," + + " type, provider, catalog_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{catalogMeta.catalogId}," + + " #{catalogMeta.catalogName}," + + " #{catalogMeta.metalakeId}," + + " #{catalogMeta.type}," + + " #{catalogMeta.provider}," + + " #{catalogMeta.catalogComment}," + + " #{catalogMeta.properties}," + + " #{catalogMeta.auditInfo}," + + " #{catalogMeta.currentVersion}," + + " #{catalogMeta.lastVersion}," + + " #{catalogMeta.deletedAt}" + + " )" + + " ON CONFLICT(catalog_id) DO UPDATE SET" + + " catalog_name = #{catalogMeta.catalogName}," + + " metalake_id = #{catalogMeta.metalakeId}," + + " type = #{catalogMeta.type}," + + " provider = #{catalogMeta.provider}," + + " catalog_comment = #{catalogMeta.catalogComment}," + + " properties = #{catalogMeta.properties}," + + " audit_info = #{catalogMeta.auditInfo}," + + " current_version = #{catalogMeta.currentVersion}," + + " last_version = #{catalogMeta.lastVersion}," + + " deleted_at = #{catalogMeta.deletedAt}"; + } + + public String updateCatalogMeta( + @Param("newCatalogMeta") CatalogPO newCatalogPO, + @Param("oldCatalogMeta") CatalogPO oldCatalogPO) { + return "UPDATE " + + TABLE_NAME + + " SET catalog_name = #{newCatalogMeta.catalogName}," + + " metalake_id = #{newCatalogMeta.metalakeId}," + + " type = #{newCatalogMeta.type}," + + " provider = #{newCatalogMeta.provider}," + + " catalog_comment = #{newCatalogMeta.catalogComment}," + + " properties = #{newCatalogMeta.properties}," + + " audit_info = #{newCatalogMeta.auditInfo}," + + " current_version = #{newCatalogMeta.currentVersion}," + + " last_version = #{newCatalogMeta.lastVersion}," + + " deleted_at = #{newCatalogMeta.deletedAt}" + + " WHERE catalog_id = #{oldCatalogMeta.catalogId}" + + " AND catalog_name = #{oldCatalogMeta.catalogName}" + + " AND metalake_id = #{oldCatalogMeta.metalakeId}" + + " AND type = #{oldCatalogMeta.type}" + + " AND provider = #{oldCatalogMeta.provider}" + + " AND (catalog_comment = #{oldCatalogMeta.catalogComment} " + + " OR (CAST(catalog_comment AS VARCHAR) IS NULL AND " + + " CAST(#{oldCatalogMeta.catalogComment} AS VARCHAR) IS NULL))" + + " AND properties = #{oldCatalogMeta.properties}" + + " AND audit_info = #{oldCatalogMeta.auditInfo}" + + " AND current_version = #{oldCatalogMeta.currentVersion}" + + " AND last_version = #{oldCatalogMeta.lastVersion}" + + " AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..b63314088cd --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetMetaPostgreSQLProvider.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper.META_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.FilesetMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.FilesetPO; + +public class FilesetMetaPostgreSQLProvider extends FilesetMetaBaseSQLProvider { + @Override + public String softDeleteFilesetMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetMetasByCatalogId(Long catalogId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetMetasBySchemaId(Long schemaId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetMetasByFilesetId(Long filesetId) { + return "UPDATE " + + META_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + @Override + public String insertFilesetMetaOnDuplicateKeyUpdate(FilesetPO filesetPO) { + return "INSERT INTO " + + META_TABLE_NAME + + "(fileset_id, fileset_name, metalake_id," + + " catalog_id, schema_id, type, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{filesetMeta.filesetId}," + + " #{filesetMeta.filesetName}," + + " #{filesetMeta.metalakeId}," + + " #{filesetMeta.catalogId}," + + " #{filesetMeta.schemaId}," + + " #{filesetMeta.type}," + + " #{filesetMeta.auditInfo}," + + " #{filesetMeta.currentVersion}," + + " #{filesetMeta.lastVersion}," + + " #{filesetMeta.deletedAt}" + + " )" + + " ON CONFLICT(fileset_id) DO UPDATE SET" + + " fileset_name = #{filesetMeta.filesetName}," + + " metalake_id = #{filesetMeta.metalakeId}," + + " catalog_id = #{filesetMeta.catalogId}," + + " schema_id = #{filesetMeta.schemaId}," + + " type = #{filesetMeta.type}," + + " audit_info = #{filesetMeta.auditInfo}," + + " current_version = #{filesetMeta.currentVersion}," + + " last_version = #{filesetMeta.lastVersion}," + + " deleted_at = #{filesetMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetVersionPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetVersionPostgreSQLProvider.java new file mode 100644 index 00000000000..55317c0c755 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/FilesetVersionPostgreSQLProvider.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper.VERSION_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.FilesetVersionBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.FilesetVersionPO; + +public class FilesetVersionPostgreSQLProvider extends FilesetVersionBaseSQLProvider { + @Override + public String softDeleteFilesetVersionsByMetalakeId(Long metalakeId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsByCatalogId(Long catalogId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsBySchemaId(Long schemaId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsByFilesetId(Long filesetId) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE fileset_id = #{filesetId} AND deleted_at = 0"; + } + + @Override + public String softDeleteFilesetVersionsByRetentionLine( + Long filesetId, long versionRetentionLine, int limit) { + return "UPDATE " + + VERSION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE fileset_id = #{filesetId} AND version <= #{versionRetentionLine} AND deleted_at = 0 LIMIT #{limit}"; + } + + @Override + public String insertFilesetVersionOnDuplicateKeyUpdate(FilesetVersionPO filesetVersionPO) { + return "INSERT INTO " + + VERSION_TABLE_NAME + + "(metalake_id, catalog_id, schema_id, fileset_id," + + " version, fileset_comment, properties, storage_location," + + " deleted_at)" + + " VALUES(" + + " #{filesetVersion.metalakeId}," + + " #{filesetVersion.catalogId}," + + " #{filesetVersion.schemaId}," + + " #{filesetVersion.filesetId}," + + " #{filesetVersion.version}," + + " #{filesetVersion.filesetComment}," + + " #{filesetVersion.properties}," + + " #{filesetVersion.storageLocation}," + + " #{filesetVersion.deletedAt}" + + " )" + + " ON CONFLICT(fileset_id, version, deleted_at) DO UPDATE SET" + + " metalake_id = #{filesetVersion.metalakeId}," + + " catalog_id = #{filesetVersion.catalogId}," + + " schema_id = #{filesetVersion.schemaId}," + + " fileset_id = #{filesetVersion.filesetId}," + + " version = #{filesetVersion.version}," + + " fileset_comment = #{filesetVersion.filesetComment}," + + " properties = #{filesetVersion.properties}," + + " storage_location = #{filesetVersion.storageLocation}," + + " deleted_at = #{filesetVersion.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..0e24b319ed8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupMetaPostgreSQLProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.GroupMetaMapper.GROUP_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.GroupMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.GroupPO; + +public class GroupMetaPostgreSQLProvider extends GroupMetaBaseSQLProvider { + @Override + public String softDeleteGroupMetaByGroupId(Long groupId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + @Override + public String softDeleteGroupMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + GROUP_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertGroupMetaOnDuplicateKeyUpdate(GroupPO groupPO) { + return "INSERT INTO " + + GROUP_TABLE_NAME + + "(group_id, group_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{groupMeta.groupId}," + + " #{groupMeta.groupName}," + + " #{groupMeta.metalakeId}," + + " #{groupMeta.auditInfo}," + + " #{groupMeta.currentVersion}," + + " #{groupMeta.lastVersion}," + + " #{groupMeta.deletedAt}" + + " )" + + " ON CONFLICT(group_id) DO UPDATE SET" + + " group_name = #{groupMeta.groupName}," + + " metalake_id = #{groupMeta.metalakeId}," + + " audit_info = #{groupMeta.auditInfo}," + + " current_version = #{groupMeta.currentVersion}," + + " last_version = #{groupMeta.lastVersion}," + + " deleted_at = #{groupMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupRoleRelPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupRoleRelPostgreSQLProvider.java new file mode 100644 index 00000000000..02f81bc73e8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/GroupRoleRelPostgreSQLProvider.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.GroupRoleRelMapper.GROUP_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.mapper.GroupRoleRelBaseSQLProvider; + +public class GroupRoleRelPostgreSQLProvider extends GroupRoleRelBaseSQLProvider { + @Override + public String softDeleteGroupRoleRelByGroupId(Long groupId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE group_id = #{groupId} AND deleted_at = 0"; + } + + @Override + public String softDeleteGroupRoleRelByGroupAndRoles(Long groupId, List roleIds) { + return ""; + } + + @Override + public String softDeleteGroupRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE group_id IN (SELECT group_id FROM " + + GROUP_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + @Override + public String softDeleteGroupRoleRelByRoleId(Long roleId) { + return "UPDATE " + + GROUP_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/MetalakeMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/MetalakeMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..5375f74c0da --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/MetalakeMetaPostgreSQLProvider.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.ibatis.annotations.Param; + +public class MetalakeMetaPostgreSQLProvider extends MetalakeMetaBaseSQLProvider { + @Override + public String softDeleteMetalakeMetaByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertMetalakeMetaOnDuplicateKeyUpdate(MetalakePO metalakePO) { + return "INSERT INTO " + + TABLE_NAME + + "(metalake_id, metalake_name, metalake_comment, properties, audit_info," + + " schema_version, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{metalakeMeta.metalakeId}," + + " #{metalakeMeta.metalakeName}," + + " #{metalakeMeta.metalakeComment}," + + " #{metalakeMeta.properties}," + + " #{metalakeMeta.auditInfo}," + + " #{metalakeMeta.schemaVersion}," + + " #{metalakeMeta.currentVersion}," + + " #{metalakeMeta.lastVersion}," + + " #{metalakeMeta.deletedAt}" + + " )" + + " ON CONFLICT(metalake_id) DO UPDATE SET" + + " metalake_name = #{metalakeMeta.metalakeName}," + + " metalake_comment = #{metalakeMeta.metalakeComment}," + + " properties = #{metalakeMeta.properties}," + + " audit_info = #{metalakeMeta.auditInfo}," + + " schema_version = #{metalakeMeta.schemaVersion}," + + " current_version = #{metalakeMeta.currentVersion}," + + " last_version = #{metalakeMeta.lastVersion}," + + " deleted_at = #{metalakeMeta.deletedAt}"; + } + + public String updateMetalakeMeta( + @Param("newMetalakeMeta") MetalakePO newMetalakePO, + @Param("oldMetalakeMeta") MetalakePO oldMetalakePO) { + return "UPDATE " + + TABLE_NAME + + " SET metalake_name = #{newMetalakeMeta.metalakeName}," + + " metalake_comment = #{newMetalakeMeta.metalakeComment}," + + " properties = #{newMetalakeMeta.properties}," + + " audit_info = #{newMetalakeMeta.auditInfo}," + + " schema_version = #{newMetalakeMeta.schemaVersion}," + + " current_version = #{newMetalakeMeta.currentVersion}," + + " last_version = #{newMetalakeMeta.lastVersion}" + + " WHERE metalake_id = #{oldMetalakeMeta.metalakeId}" + + " AND metalake_name = #{oldMetalakeMeta.metalakeName}" + + " AND (metalake_comment = #{oldMetalakeMeta.metalakeComment} " + + " OR (CAST(metalake_comment AS VARCHAR) IS NULL AND " + + " CAST(#{oldMetalakeMeta.metalakeComment} AS VARCHAR) IS NULL))" + + " AND properties = #{oldMetalakeMeta.properties}" + + " AND audit_info = #{oldMetalakeMeta.auditInfo}" + + " AND schema_version = #{oldMetalakeMeta.schemaVersion}" + + " AND current_version = #{oldMetalakeMeta.currentVersion}" + + " AND last_version = #{oldMetalakeMeta.lastVersion}" + + " AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/OwnerMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/OwnerMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..6374a3fbd28 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/OwnerMetaPostgreSQLProvider.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper.OWNER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; +import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; +import org.apache.gravitino.storage.relational.mapper.OwnerMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; + +public class OwnerMetaPostgreSQLProvider extends OwnerMetaBaseSQLProvider { + @Override + public String softDeleteOwnerRelByMetadataObjectIdAndType( + Long metadataObjectId, String metadataObjectType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metadata_object_id = #{metadataObjectId} AND metadata_object_type = #{metadataObjectType} AND deleted_at = 0"; + } + + @Override + public String softDeleteOwnerRelByOwnerIdAndType(Long ownerId, String ownerType) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE owner_id = #{ownerId} AND owner_type = #{ownerType} AND deleted_at = 0"; + } + + @Override + public String softDeleteOwnerRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at =0"; + } + + @Override + public String softDeleteOwnerRelByCatalogId(Long catalogId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE EXISTS (" + + " SELECT ct.catalog_id FROM " + + CatalogMetaMapper.TABLE_NAME + + " ct WHERE ct.catalog_id = #{catalogId} AND ct.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'" + + " UNION " + + " SELECT st.catalog_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.catalog_id = #{catalogId} AND st.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.catalog_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.catalog_id = #{catalogId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.catalog_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.catalog_id = #{catalogId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.catalog_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.catalog_id = #{catalogId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND" + + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } + + @Override + public String sotDeleteOwnerRelBySchemaId(Long schemaId) { + return "UPDATE " + + OWNER_TABLE_NAME + + " ot SET deleted_at = floor(extract(epoch from((current_timestamp - timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE EXISTS (" + + " SELECT st.schema_id FROM " + + SchemaMetaMapper.TABLE_NAME + + " st WHERE st.schema_id = #{schemaId} AND st.deleted_at = 0 AND ot.deleted_at = 0 " + + "AND st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'" + + " UNION " + + " SELECT tt.schema_id FROM " + + TopicMetaMapper.TABLE_NAME + + " tt WHERE tt.schema_id = #{schemaId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'" + + " UNION " + + " SELECT tat.schema_id FROM " + + TableMetaMapper.TABLE_NAME + + " tat WHERE tat.schema_id = #{schemaId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'" + + " UNION " + + " SELECT ft.schema_id FROM " + + FilesetMetaMapper.META_TABLE_NAME + + " ft WHERE ft.schema_id = #{schemaId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND " + + "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'" + + ")"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/RoleMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/RoleMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..72e5808e37d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/RoleMetaPostgreSQLProvider.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.RoleMetaMapper.ROLE_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.RoleMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.RolePO; + +public class RoleMetaPostgreSQLProvider extends RoleMetaBaseSQLProvider { + @Override + public String softDeleteRoleMetaByRoleId(Long roleId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + @Override + public String softDeleteRoleMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + ROLE_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertRoleMetaOnDuplicateKeyUpdate(RolePO rolePO) { + return "INSERT INTO " + + ROLE_TABLE_NAME + + "(role_id, role_name," + + " metalake_id, properties," + + " audit_info, current_version, last_version, deleted_at)" + + " VALUES(" + + " #{roleMeta.roleId}," + + " #{roleMeta.roleName}," + + " #{roleMeta.metalakeId}," + + " #{roleMeta.properties}," + + " #{roleMeta.auditInfo}," + + " #{roleMeta.currentVersion}," + + " #{roleMeta.lastVersion}," + + " #{roleMeta.deletedAt}" + + " ) ON CONFLICT (role_id) DO UPDATE SET" + + " role_name = #{roleMeta.roleName}," + + " metalake_id = #{roleMeta.metalakeId}," + + " properties = #{roleMeta.properties}," + + " audit_info = #{roleMeta.auditInfo}," + + " current_version = #{roleMeta.currentVersion}," + + " last_version = #{roleMeta.lastVersion}," + + " deleted_at = #{roleMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SchemaMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SchemaMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..c2a565ddd08 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SchemaMetaPostgreSQLProvider.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.SchemaMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.SchemaPO; + +public class SchemaMetaPostgreSQLProvider extends SchemaMetaBaseSQLProvider { + @Override + public String insertSchemaMetaOnDuplicateKeyUpdate(SchemaPO schemaPO) { + return "INSERT INTO " + + TABLE_NAME + + "(schema_id, schema_name, metalake_id," + + " catalog_id, schema_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{schemaMeta.schemaId}," + + " #{schemaMeta.schemaName}," + + " #{schemaMeta.metalakeId}," + + " #{schemaMeta.catalogId}," + + " #{schemaMeta.schemaComment}," + + " #{schemaMeta.properties}," + + " #{schemaMeta.auditInfo}," + + " #{schemaMeta.currentVersion}," + + " #{schemaMeta.lastVersion}," + + " #{schemaMeta.deletedAt}" + + " )" + + " ON CONFLICT(schema_id) DO UPDATE SET " + + " schema_name = #{schemaMeta.schemaName}," + + " metalake_id = #{schemaMeta.metalakeId}," + + " catalog_id = #{schemaMeta.catalogId}," + + " schema_comment = #{schemaMeta.schemaComment}," + + " properties = #{schemaMeta.properties}," + + " audit_info = #{schemaMeta.auditInfo}," + + " current_version = #{schemaMeta.currentVersion}," + + " last_version = #{schemaMeta.lastVersion}," + + " deleted_at = #{schemaMeta.deletedAt}"; + } + + @Override + public String softDeleteSchemaMetasBySchemaId(Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String softDeleteSchemaMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteSchemaMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SecurableObjectPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SecurableObjectPostgreSQLProvider.java new file mode 100644 index 00000000000..ff1340e6c2c --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/SecurableObjectPostgreSQLProvider.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.ROLE_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.SecurableObjectBaseSQLProvider; + +public class SecurableObjectPostgreSQLProvider extends SecurableObjectBaseSQLProvider { + @Override + public String softDeleteSecurableObjectsByRoleId(Long roleId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + @Override + public String softDeleteRoleMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " ob SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE exists (SELECT * from " + + ROLE_TABLE_NAME + + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" + + " AND ro.deleted_at = 0) AND ob.deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TableMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TableMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..d1f5d398591 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TableMetaPostgreSQLProvider.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.TableMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.TablePO; + +public class TableMetaPostgreSQLProvider extends TableMetaBaseSQLProvider { + @Override + public String insertTableMetaOnDuplicateKeyUpdate(TablePO tablePO) { + return "INSERT INTO " + + TABLE_NAME + + "(table_id, table_name, metalake_id," + + " catalog_id, schema_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tableMeta.tableId}," + + " #{tableMeta.tableName}," + + " #{tableMeta.metalakeId}," + + " #{tableMeta.catalogId}," + + " #{tableMeta.schemaId}," + + " #{tableMeta.auditInfo}," + + " #{tableMeta.currentVersion}," + + " #{tableMeta.lastVersion}," + + " #{tableMeta.deletedAt}" + + " )" + + " ON CONFLICT (table_id) DO UPDATE SET " + + " table_name = #{tableMeta.tableName}," + + " metalake_id = #{tableMeta.metalakeId}," + + " catalog_id = #{tableMeta.catalogId}," + + " schema_id = #{tableMeta.schemaId}," + + " audit_info = #{tableMeta.auditInfo}," + + " current_version = #{tableMeta.currentVersion}," + + " last_version = #{tableMeta.lastVersion}," + + " deleted_at = #{tableMeta.deletedAt}"; + } + + @Override + public String softDeleteTableMetasByTableId(Long tableId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE table_id = #{tableId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTableMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTableMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTableMetasBySchemaId(Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..75a3863f7ba --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetaPostgreSQLProvider.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TagMetaMapper.TAG_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.TagPO; +import org.apache.ibatis.annotations.Param; + +public class TagMetaPostgreSQLProvider extends TagMetaBaseSQLProvider { + @Override + public String softDeleteTagMetaByMetalakeAndTagName(String metalakeName, String tagName) { + return "UPDATE " + + TAG_TABLE_NAME + + " tm SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE tm.metalake_id IN (" + + " SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0"; + } + + @Override + public String softDeleteTagMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertTagMetaOnDuplicateKeyUpdate(TagPO tagPO) { + return "INSERT INTO " + + TAG_TABLE_NAME + + "(tag_id, tag_name," + + " metalake_id, tag_comment, properties, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{tagMeta.tagId}," + + " #{tagMeta.tagName}," + + " #{tagMeta.metalakeId}," + + " #{tagMeta.comment}," + + " #{tagMeta.properties}," + + " #{tagMeta.auditInfo}," + + " #{tagMeta.currentVersion}," + + " #{tagMeta.lastVersion}," + + " #{tagMeta.deletedAt}" + + " )" + + " ON CONFLICT(tag_id) DO UPDATE SET" + + " tag_name = #{tagMeta.tagName}," + + " metalake_id = #{tagMeta.metalakeId}," + + " tag_comment = #{tagMeta.comment}," + + " properties = #{tagMeta.properties}," + + " audit_info = #{tagMeta.auditInfo}," + + " current_version = #{tagMeta.currentVersion}," + + " last_version = #{tagMeta.lastVersion}," + + " deleted_at = #{tagMeta.deletedAt}"; + } + + public String updateTagMeta( + @Param("newTagMeta") TagPO newTagPO, @Param("oldTagMeta") TagPO oldTagPO) { + return "UPDATE " + + TAG_TABLE_NAME + + " SET tag_name = #{newTagMeta.tagName}," + + " tag_comment = #{newTagMeta.comment}," + + " properties = #{newTagMeta.properties}," + + " audit_info = #{newTagMeta.auditInfo}," + + " current_version = #{newTagMeta.currentVersion}," + + " last_version = #{newTagMeta.lastVersion}," + + " deleted_at = #{newTagMeta.deletedAt}" + + " WHERE tag_id = #{oldTagMeta.tagId}" + + " AND metalake_id = #{oldTagMeta.metalakeId}" + + " AND tag_name = #{oldTagMeta.tagName}" + + " AND (tag_comment = #{oldTagMeta.comment} " + + " OR (CAST(tag_comment AS VARCHAR) IS NULL AND CAST(#{oldTagMeta.comment} AS VARCHAR) IS NULL))" + + " AND properties = #{oldTagMeta.properties}" + + " AND audit_info = #{oldTagMeta.auditInfo}" + + " AND current_version = #{oldTagMeta.currentVersion}" + + " AND last_version = #{oldTagMeta.lastVersion}" + + " AND deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetadataObjectRelPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetadataObjectRelPostgreSQLProvider.java new file mode 100644 index 00000000000..1bb716248be --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TagMetadataObjectRelPostgreSQLProvider.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper.TAG_METADATA_OBJECT_RELATION_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetaMapper; +import org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelBaseSQLProvider; + +public class TagMetadataObjectRelPostgreSQLProvider extends TagMetadataObjectRelBaseSQLProvider { + @Override + public String softDeleteTagMetadataObjectRelsByMetalakeAndTagName( + String metalakeName, String tagName) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " + + MetalakeMetaMapper.TABLE_NAME + + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + @Override + public String softDeleteTagMetadataObjectRelsByMetalakeId(Long metalakeId) { + return "UPDATE " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE EXISTS (SELECT * FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0"; + } + + @Override + public String batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + Long metadataObjectId, String metadataObjectType, List tagIds) { + return ""; + } + + @Override + public String listTagMetadataObjectRelsByMetalakeAndTagName(String metalakeName, String tagName) { + return "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," + + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," + + " te.current_version as currentVersion, te.last_version as lastVersion," + + " te.deleted_at as deletedAt" + + " FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te JOIN " + + TagMetaMapper.TAG_TABLE_NAME + + " tm ON te.tag_id = tm.tag_id JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TopicMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TopicMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..ef7e45cbc1e --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/TopicMetaPostgreSQLProvider.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.TopicMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.TopicPO; + +public class TopicMetaPostgreSQLProvider extends TopicMetaBaseSQLProvider { + + @Override + public String softDeleteTopicMetasByTopicId(Long topicId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE topic_id = #{topicId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTopicMetasByCatalogId(Long catalogId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTopicMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000)))" + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String softDeleteTopicMetasBySchemaId(Long schemaId) { + return "UPDATE " + + TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE schema_id = #{schemaId} AND deleted_at = 0"; + } + + @Override + public String insertTopicMetaOnDuplicateKeyUpdate(TopicPO topicPO) { + return "INSERT INTO " + + TABLE_NAME + + "(topic_id, topic_name, metalake_id, catalog_id, schema_id," + + " comment, properties, audit_info, current_version, last_version," + + " deleted_at)" + + " VALUES(" + + " #{topicMeta.topicId}," + + " #{topicMeta.topicName}," + + " #{topicMeta.metalakeId}," + + " #{topicMeta.catalogId}," + + " #{topicMeta.schemaId}," + + " #{topicMeta.comment}," + + " #{topicMeta.properties}," + + " #{topicMeta.auditInfo}," + + " #{topicMeta.currentVersion}," + + " #{topicMeta.lastVersion}," + + " #{topicMeta.deletedAt}" + + " )" + + " ON CONFLICT (topic_id) DO UPDATE SET" + + " topic_name = #{topicMeta.topicName}," + + " metalake_id = #{topicMeta.metalakeId}," + + " catalog_id = #{topicMeta.catalogId}," + + " schema_id = #{topicMeta.schemaId}," + + " comment = #{topicMeta.comment}," + + " properties = #{topicMeta.properties}," + + " audit_info = #{topicMeta.auditInfo}," + + " current_version = #{topicMeta.currentVersion}," + + " last_version = #{topicMeta.lastVersion}," + + " deleted_at = #{topicMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserMetaPostgreSQLProvider.java new file mode 100644 index 00000000000..af7d65d2ab5 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserMetaPostgreSQLProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import org.apache.gravitino.storage.relational.mapper.UserMetaBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.UserPO; + +public class UserMetaPostgreSQLProvider extends UserMetaBaseSQLProvider { + @Override + public String softDeleteUserMetaByUserId(Long userId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + @Override + public String softDeleteUserMetasByMetalakeId(Long metalakeId) { + return "UPDATE " + + USER_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0"; + } + + @Override + public String insertUserMetaOnDuplicateKeyUpdate(UserPO userPO) { + return "INSERT INTO " + + USER_TABLE_NAME + + "(user_id, user_name," + + "metalake_id, audit_info," + + " current_version, last_version, deleted_at)" + + " VALUES(" + + " #{userMeta.userId}," + + " #{userMeta.userName}," + + " #{userMeta.metalakeId}," + + " #{userMeta.auditInfo}," + + " #{userMeta.currentVersion}," + + " #{userMeta.lastVersion}," + + " #{userMeta.deletedAt}" + + " )" + + " ON CONFLICT(user_id) DO UPDATE SET" + + " user_name = #{userMeta.userName}," + + " metalake_id = #{userMeta.metalakeId}," + + " audit_info = #{userMeta.auditInfo}," + + " current_version = #{userMeta.currentVersion}," + + " last_version = #{userMeta.lastVersion}," + + " deleted_at = #{userMeta.deletedAt}"; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserRoleRelPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserRoleRelPostgreSQLProvider.java new file mode 100644 index 00000000000..dddca1efc17 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/postgresql/UserRoleRelPostgreSQLProvider.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.storage.relational.mapper.postgresql; + +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_ROLE_RELATION_TABLE_NAME; +import static org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper.USER_TABLE_NAME; + +import java.util.List; +import org.apache.gravitino.storage.relational.mapper.UserRoleRelBaseSQLProvider; +import org.apache.gravitino.storage.relational.po.UserRoleRelPO; + +public class UserRoleRelPostgreSQLProvider extends UserRoleRelBaseSQLProvider { + @Override + public String softDeleteUserRoleRelByUserId(Long userId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE user_id = #{userId} AND deleted_at = 0"; + } + + @Override + public String softDeleteUserRoleRelByUserAndRoles(Long userId, List roleIds) { + return ""; + } + + @Override + public String softDeleteUserRoleRelByMetalakeId(Long metalakeId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE user_id IN (SELECT user_id FROM " + + USER_TABLE_NAME + + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0)" + + " AND deleted_at = 0"; + } + + @Override + public String softDeleteUserRoleRelByRoleId(Long roleId) { + return "UPDATE " + + USER_ROLE_RELATION_TABLE_NAME + + " SET deleted_at = floor(extract(epoch from((current_timestamp -" + + " timestamp '1970-01-01 00:00:00')*1000))) " + + " WHERE role_id = #{roleId} AND deleted_at = 0"; + } + + @Override + public String batchInsertUserRoleRelOnDuplicateKeyUpdate(List userRoleRelPOs) { + return ""; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java index 2ffc10dac59..2762d55a7fd 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java @@ -22,10 +22,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -41,10 +39,10 @@ import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.po.GroupPO; import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; -import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; /** The service class for group metadata. It provides the basic database operations for group. */ public class GroupMetaService { @@ -92,9 +90,9 @@ public GroupEntity getGroupByIdentifier(NameIdentifier identifier) { Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); GroupPO groupPO = getGroupPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = RoleMetaService.getInstance().listRolesByGroupId(groupPO.getGroupId()); - return POConverters.fromGroupPO(groupPO, rolePOs, identifier.namespace()); + return POConverters.fromGroupPO( + groupPO, SupplierUtils.createRolePOsSupplier(groupPO), identifier.namespace()); } public List listGroupsByRoleIdent(NameIdentifier roleIdent) { @@ -107,7 +105,7 @@ public List listGroupsByRoleIdent(NameIdentifier roleIdent) { po -> POConverters.fromGroupPO( po, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(po), AuthorizationUtils.ofGroupNamespace(roleIdent.namespace().level(0)))) .collect(Collectors.toList()); } @@ -121,7 +119,8 @@ public void insertGroup(GroupEntity groupEntity, boolean overwritten) throws IOE GroupPO.Builder builder = GroupPO.builder().withMetalakeId(metalakeId); GroupPO GroupPO = POConverters.initializeGroupPOWithVersion(groupEntity, builder); - List roleIds = Optional.ofNullable(groupEntity.roleIds()).orElse(Lists.newArrayList()); + List roleIds = + groupEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toList()); List groupRoleRelPOS = POConverters.initializeGroupRoleRelsPOWithVersion(groupEntity, roleIds); @@ -186,10 +185,10 @@ public GroupEntity updateGroup( Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); GroupPO oldGroupPO = getGroupPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = - RoleMetaService.getInstance().listRolesByGroupId(oldGroupPO.getGroupId()); + GroupEntity oldGroupEntity = - POConverters.fromGroupPO(oldGroupPO, rolePOs, identifier.namespace()); + POConverters.fromGroupPO( + oldGroupPO, SupplierUtils.createRolePOsSupplier(oldGroupPO), identifier.namespace()); GroupEntity newEntity = (GroupEntity) updater.apply((E) oldGroupEntity); Preconditions.checkArgument( @@ -199,11 +198,10 @@ public GroupEntity updateGroup( oldGroupEntity.id()); Set oldRoleIds = - oldGroupEntity.roleIds() == null - ? Sets.newHashSet() - : Sets.newHashSet(oldGroupEntity.roleIds()); + oldGroupEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); + Set newRoleIds = - newEntity.roleIds() == null ? Sets.newHashSet() : Sets.newHashSet(newEntity.roleIds()); + newEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); Set insertRoleIds = Sets.difference(newRoleIds, oldRoleIds); Set deleteRoleIds = Sets.difference(oldRoleIds, newRoleIds); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java index 1118467b0a7..502b427f045 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.storage.relational.service; -import java.util.Collections; import java.util.Optional; import org.apache.gravitino.Entity; import org.apache.gravitino.MetadataObject; @@ -30,6 +29,7 @@ import org.apache.gravitino.storage.relational.po.UserPO; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; import org.apache.gravitino.utils.NameIdentifierUtil; /** This class is an utilization class to retrieve owner relation. */ @@ -58,7 +58,7 @@ public Optional getOwner(NameIdentifier identifier, Entity.EntityType ty return Optional.of( POConverters.fromUserPO( userPO, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(userPO), AuthorizationUtils.ofUserNamespace(NameIdentifierUtil.getMetalake(identifier)))); } @@ -71,7 +71,7 @@ public Optional getOwner(NameIdentifier identifier, Entity.EntityType ty return Optional.of( POConverters.fromGroupPO( groupPO, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(groupPO), AuthorizationUtils.ofGroupNamespace(NameIdentifierUtil.getMetalake(identifier)))); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 4560b74e0d0..743eb6338d2 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -39,14 +39,12 @@ import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; import org.apache.gravitino.utils.NameIdentifierUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** The service class for role metadata. It provides the basic database operations for role. */ public class RoleMetaService { - private static final Logger LOG = LoggerFactory.getLogger(RoleMetaService.class); private static final RoleMetaService INSTANCE = new RoleMetaService(); public static RoleMetaService getInstance() { @@ -109,33 +107,12 @@ public List listRolesByMetadataObjectIdentAndType( .map( po -> POConverters.fromRolePO( - po, listSecurableObjects(po), AuthorizationUtils.ofRoleNamespace(metalake))) + po, + SupplierUtils.createSecurableObjectsSupplier(po), + AuthorizationUtils.ofRoleNamespace(metalake))) .collect(Collectors.toList()); } - private List listSecurableObjects(RolePO po) { - List securableObjectPOs = listSecurableObjectsByRoleId(po.getRoleId()); - List securableObjects = Lists.newArrayList(); - - for (SecurableObjectPO securableObjectPO : securableObjectPOs) { - String fullName = - MetadataObjectService.getMetadataObjectFullName( - securableObjectPO.getType(), securableObjectPO.getMetadataObjectId()); - if (fullName != null) { - securableObjects.add( - POConverters.fromSecurableObjectPO( - fullName, securableObjectPO, getType(securableObjectPO.getType()))); - } else { - LOG.info( - "The securable object {} {} may be deleted", - securableObjectPO.getMetadataObjectId(), - securableObjectPO.getType()); - } - } - - return securableObjects; - } - public List listRolesByGroupId(Long groupId) { return SessionUtils.getWithoutCommit( RoleMetaMapper.class, mapper -> mapper.listRolesByGroupId(groupId)); @@ -197,9 +174,8 @@ public RoleEntity getRoleByIdentifier(NameIdentifier identifier) { MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); RolePO rolePO = getRolePOByMetalakeIdAndName(metalakeId, identifier.name()); - List securableObjects = listSecurableObjects(rolePO); - - return POConverters.fromRolePO(rolePO, securableObjects, identifier.namespace()); + return POConverters.fromRolePO( + rolePO, SupplierUtils.createSecurableObjectsSupplier(rolePO), identifier.namespace()); } public boolean deleteRole(NameIdentifier identifier) { @@ -232,7 +208,7 @@ public boolean deleteRole(NameIdentifier identifier) { return true; } - private List listSecurableObjectsByRoleId(Long roleId) { + public List listSecurableObjectsByRoleId(Long roleId) { return SessionUtils.getWithoutCommit( SecurableObjectMapper.class, mapper -> mapper.listSecurableObjectsByRoleId(roleId)); } @@ -273,10 +249,6 @@ public int deleteRoleMetasByLegacyTimeline(long legacyTimeline, int limit) { + securableObjectsCount[0]; } - private MetadataObject.Type getType(String type) { - return MetadataObject.Type.valueOf(type); - } - private String getEntityType(SecurableObject securableObject) { return securableObject.type().name(); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index f89c4c5f0f0..3e116781162 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -211,7 +211,8 @@ public boolean deleteSchema(NameIdentifier identifier, boolean cascade) { mapper -> mapper.softDeleteTopicMetasBySchemaId(schemaId)), () -> SessionUtils.doWithoutCommit( - OwnerMetaMapper.class, mapper -> mapper.sotDeleteOwnerRelBySchemaId(schemaId))); + OwnerMetaMapper.class, + mapper -> mapper.softDeleteOwnerRelBySchemaId(schemaId))); } else { List tableEntities = TableMetaService.getInstance() diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java index e7d0a435a1b..67bbfbce3df 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java @@ -22,10 +22,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -39,12 +37,12 @@ import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.mapper.UserMetaMapper; import org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper; -import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.po.UserPO; import org.apache.gravitino.storage.relational.po.UserRoleRelPO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.storage.relational.utils.SupplierUtils; /** The service class for user metadata. It provides the basic database operations for user. */ public class UserMetaService { @@ -92,9 +90,9 @@ public UserEntity getUserByIdentifier(NameIdentifier identifier) { Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); UserPO userPO = getUserPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = RoleMetaService.getInstance().listRolesByUserId(userPO.getUserId()); - return POConverters.fromUserPO(userPO, rolePOs, identifier.namespace()); + return POConverters.fromUserPO( + userPO, SupplierUtils.createRolePOsSupplier(userPO), identifier.namespace()); } public List listUsersByRoleIdent(NameIdentifier roleIdent) { @@ -107,7 +105,7 @@ public List listUsersByRoleIdent(NameIdentifier roleIdent) { po -> POConverters.fromUserPO( po, - Collections.emptyList(), + SupplierUtils.createRolePOsSupplier(po), AuthorizationUtils.ofUserNamespace(roleIdent.namespace().level(0)))) .collect(Collectors.toList()); } @@ -121,7 +119,8 @@ public void insertUser(UserEntity userEntity, boolean overwritten) throws IOExce UserPO.Builder builder = UserPO.builder().withMetalakeId(metalakeId); UserPO userPO = POConverters.initializeUserPOWithVersion(userEntity, builder); - List roleIds = Optional.ofNullable(userEntity.roleIds()).orElse(Lists.newArrayList()); + List roleIds = + userEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toList()); List userRoleRelPOs = POConverters.initializeUserRoleRelsPOWithVersion(userEntity, roleIds); @@ -185,8 +184,9 @@ public UserEntity updateUser( Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); UserPO oldUserPO = getUserPOByMetalakeIdAndName(metalakeId, identifier.name()); - List rolePOs = RoleMetaService.getInstance().listRolesByUserId(oldUserPO.getUserId()); - UserEntity oldUserEntity = POConverters.fromUserPO(oldUserPO, rolePOs, identifier.namespace()); + UserEntity oldUserEntity = + POConverters.fromUserPO( + oldUserPO, SupplierUtils.createRolePOsSupplier(oldUserPO), identifier.namespace()); UserEntity newEntity = (UserEntity) updater.apply((E) oldUserEntity); Preconditions.checkArgument( @@ -196,11 +196,10 @@ public UserEntity updateUser( oldUserEntity.id()); Set oldRoleIds = - oldUserEntity.roleIds() == null - ? Sets.newHashSet() - : Sets.newHashSet(oldUserEntity.roleIds()); + oldUserEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); + Set newRoleIds = - newEntity.roleIds() == null ? Sets.newHashSet() : Sets.newHashSet(newEntity.roleIds()); + newEntity.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet()); Set insertRoleIds = Sets.difference(newRoleIds, oldRoleIds); Set deleteRoleIds = Sets.difference(oldRoleIds, newRoleIds); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index 684252f17ee..9d928271623 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -26,6 +26,7 @@ import org.apache.commons.pool2.impl.BaseObjectPoolConfig; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; +import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType; import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper; @@ -73,7 +74,9 @@ private SqlSessionFactoryHelper() {} public void init(Config config) { // Initialize the data source BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL)); + String jdbcUrl = config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL); + JDBCBackendType jdbcType = JDBCBackendType.fromURI(jdbcUrl); + dataSource.setUrl(jdbcUrl); dataSource.setDriverClassName(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)); dataSource.setUsername(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER)); dataSource.setPassword(config.get(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)); @@ -102,6 +105,7 @@ public void init(Config config) { // Initialize the configuration Configuration configuration = new Configuration(environment); + configuration.setDatabaseId(jdbcType.name().toLowerCase()); configuration.addMapper(MetalakeMetaMapper.class); configuration.addMapper(CatalogMetaMapper.class); configuration.addMapper(SchemaMetaMapper.class); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java index 82d739a41cc..c21ef1df08e 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java @@ -24,11 +24,13 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.Namespace; +import org.apache.gravitino.authorization.AuthorizationUtils; import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; @@ -699,29 +701,31 @@ public static UserPO updateUserPOWithVersion(UserPO oldUserPO, UserEntity newUse * Convert {@link UserPO} to {@link UserEntity} * * @param userPO UserPo object to be converted - * @param rolePOs list of rolePO + * @param rolePOsSupplier Supplier for the list of rolePO * @param namespace Namespace object to be associated with the user * @return UserEntity object from UserPO object */ - public static UserEntity fromUserPO(UserPO userPO, List rolePOs, Namespace namespace) { + public static UserEntity fromUserPO( + UserPO userPO, Supplier> rolePOsSupplier, Namespace namespace) { try { - List roleNames = - rolePOs.stream().map(RolePO::getRoleName).collect(Collectors.toList()); - List roleIds = rolePOs.stream().map(RolePO::getRoleId).collect(Collectors.toList()); - UserEntity.Builder builder = UserEntity.builder() .withId(userPO.getUserId()) .withName(userPO.getUserName()) .withNamespace(namespace) .withAuditInfo( - JsonUtils.anyFieldMapper().readValue(userPO.getAuditInfo(), AuditInfo.class)); - if (!roleNames.isEmpty()) { - builder.withRoleNames(roleNames); - } - if (!roleIds.isEmpty()) { - builder.withRoleIds(roleIds); - } + JsonUtils.anyFieldMapper().readValue(userPO.getAuditInfo(), AuditInfo.class)) + .withRolesSupplier( + () -> + rolePOsSupplier.get().stream() + .map( + po -> + fromRolePO( + po, + SupplierUtils.createSecurableObjectsSupplier(po), + AuthorizationUtils.ofRoleNamespace(namespace.level(0)))) + .collect(Collectors.toList())); + return builder.build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to deserialize json object:", e); @@ -732,30 +736,31 @@ public static UserEntity fromUserPO(UserPO userPO, List rolePOs, Namespa * Convert {@link GroupPO} to {@link GroupEntity} * * @param groupPO GroupPO object to be converted - * @param rolePOs list of rolePO + * @param rolePOsSupplier Supplier for the list of rolePO * @param namespace Namespace object to be associated with the group * @return GroupEntity object from GroupPO object */ public static GroupEntity fromGroupPO( - GroupPO groupPO, List rolePOs, Namespace namespace) { + GroupPO groupPO, Supplier> rolePOsSupplier, Namespace namespace) { try { - List roleNames = - rolePOs.stream().map(RolePO::getRoleName).collect(Collectors.toList()); - List roleIds = rolePOs.stream().map(RolePO::getRoleId).collect(Collectors.toList()); - GroupEntity.Builder builder = GroupEntity.builder() .withId(groupPO.getGroupId()) .withName(groupPO.getGroupName()) .withNamespace(namespace) .withAuditInfo( - JsonUtils.anyFieldMapper().readValue(groupPO.getAuditInfo(), AuditInfo.class)); - if (!roleNames.isEmpty()) { - builder.withRoleNames(roleNames); - } - if (!roleIds.isEmpty()) { - builder.withRoleIds(roleIds); - } + JsonUtils.anyFieldMapper().readValue(groupPO.getAuditInfo(), AuditInfo.class)) + .withRolesSupplier( + () -> + rolePOsSupplier.get().stream() + .map( + po -> + fromRolePO( + po, + SupplierUtils.createSecurableObjectsSupplier(po), + AuthorizationUtils.ofRoleNamespace(namespace.level(0)))) + .collect(Collectors.toList())); + return builder.build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to deserialize json object:", e); @@ -918,14 +923,16 @@ public static SecurableObject fromSecurableObjectPO( } public static RoleEntity fromRolePO( - RolePO rolePO, List securableObjects, Namespace namespace) { + RolePO rolePO, + Supplier> securableObjectsSupplier, + Namespace namespace) { try { return RoleEntity.builder() .withId(rolePO.getRoleId()) .withName(rolePO.getRoleName()) .withNamespace(namespace) .withProperties(JsonUtils.anyFieldMapper().readValue(rolePO.getProperties(), Map.class)) - .withSecurableObjects(securableObjects) + .withSecurableObjectsSupplier(securableObjectsSupplier) .withAuditInfo( JsonUtils.anyFieldMapper().readValue(rolePO.getAuditInfo(), AuditInfo.class)) .build(); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/utils/SupplierUtils.java b/core/src/main/java/org/apache/gravitino/storage/relational/utils/SupplierUtils.java new file mode 100644 index 00000000000..16c662c476d --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/utils/SupplierUtils.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.utils; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.function.Supplier; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.storage.relational.po.GroupPO; +import org.apache.gravitino.storage.relational.po.RolePO; +import org.apache.gravitino.storage.relational.po.SecurableObjectPO; +import org.apache.gravitino.storage.relational.po.UserPO; +import org.apache.gravitino.storage.relational.service.MetadataObjectService; +import org.apache.gravitino.storage.relational.service.RoleMetaService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* This class is a utilization class for creating kinds of suppliers */ +public class SupplierUtils { + + private static final Logger LOG = LoggerFactory.getLogger(SupplierUtils.class); + + private SupplierUtils() {} + + public static Supplier> createRolePOsSupplier(UserPO userPO) { + return new Supplier>() { + private List rolePOs; + private boolean waitToLoad = true; + + @Override + public List get() { + if (waitToLoad) { + rolePOs = RoleMetaService.getInstance().listRolesByUserId(userPO.getUserId()); + waitToLoad = false; + } + + return rolePOs; + } + }; + } + + public static Supplier> createRolePOsSupplier(GroupPO groupPO) { + return new Supplier>() { + private List rolePOS; + private boolean waitToLoad = true; + + @Override + public List get() { + if (waitToLoad) { + rolePOS = RoleMetaService.getInstance().listRolesByGroupId(groupPO.getGroupId()); + waitToLoad = false; + } + + return rolePOS; + } + }; + } + + public static Supplier> createSecurableObjectsSupplier(RolePO rolePO) { + return new Supplier>() { + private List securableObjects; + + private boolean waitToLoad = true; + + @Override + public List get() { + if (waitToLoad) { + List securableObjectPOs = + RoleMetaService.getInstance().listSecurableObjectsByRoleId(rolePO.getRoleId()); + + securableObjects = Lists.newArrayList(); + + for (SecurableObjectPO securableObjectPO : securableObjectPOs) { + String fullName = + MetadataObjectService.getMetadataObjectFullName( + securableObjectPO.getType(), securableObjectPO.getMetadataObjectId()); + if (fullName != null) { + securableObjects.add( + POConverters.fromSecurableObjectPO( + fullName, + securableObjectPO, + MetadataObject.Type.valueOf(securableObjectPO.getType()))); + } else { + LOG.info( + "The securable object {} {} may be deleted", + securableObjectPO.getMetadataObjectId(), + securableObjectPO.getType()); + } + } + waitToLoad = false; + } + + return securableObjects; + } + }; + } +} diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java index ce3c4e90c9d..946cebf972e 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -157,7 +157,7 @@ public void testGrantRoleToUser() { String notExist = "not-exist"; User user = accessControlManager.getUser(METALAKE, USER); - Assertions.assertNull(user.roles()); + Assertions.assertTrue(user.roles().isEmpty()); reset(authorizationPlugin); diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java b/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java index 83a562f640d..0b0b93ef056 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestOwnerManager.java @@ -125,8 +125,7 @@ public static void setUp() throws IOException, IllegalAccessException { UserEntity.builder() .withId(idGenerator.nextId()) .withName(USER) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withNamespace(AuthorizationUtils.ofUserNamespace(METALAKE)) .withAuditInfo(audit) .build(); @@ -136,8 +135,7 @@ public static void setUp() throws IOException, IllegalAccessException { GroupEntity.builder() .withId(idGenerator.nextId()) .withName(GROUP) - .withRoleNames(Collections.emptyList()) - .withRoleIds(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withNamespace(AuthorizationUtils.ofUserNamespace(METALAKE)) .withAuditInfo(audit) .build(); diff --git a/core/src/test/java/org/apache/gravitino/meta/TestEntity.java b/core/src/test/java/org/apache/gravitino/meta/TestEntity.java index f4c32dca718..2e24a7eb013 100644 --- a/core/src/test/java/org/apache/gravitino/meta/TestEntity.java +++ b/core/src/test/java/org/apache/gravitino/meta/TestEntity.java @@ -21,10 +21,15 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.time.Instant; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import org.apache.gravitino.Catalog; import org.apache.gravitino.Field; import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.file.Fileset; import org.junit.jupiter.api.Assertions; @@ -237,19 +242,48 @@ public void testUser() { .withId(userId) .withName(userName) .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) + .withRoles( + Lists.newArrayList( + RoleEntity.builder() + .withId(1L) + .withName("role") + .withAuditInfo(auditInfo) + .build())) .build(); Map fields = testUserEntity.fields(); Assertions.assertEquals(userId, fields.get(UserEntity.ID)); Assertions.assertEquals(userName, fields.get(UserEntity.NAME)); Assertions.assertEquals(auditInfo, fields.get(UserEntity.AUDIT_INFO)); - Assertions.assertEquals(Lists.newArrayList("role"), fields.get(UserEntity.ROLE_NAMES)); + Assertions.assertEquals( + Lists.newArrayList( + RoleEntity.builder().withId(1L).withName("role").withAuditInfo(auditInfo).build()), + ((Supplier>) fields.get(UserEntity.ROLES_SUPPLIER)).get()); UserEntity testUserEntityWithoutFields = UserEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); - Assertions.assertNull(testUserEntityWithoutFields.roles()); + Assertions.assertTrue(testUserEntityWithoutFields.roles().isEmpty()); + + // Test lazily loading + AtomicBoolean hasCallRolesSupplier = new AtomicBoolean(false); + + UserEntity userWithSupplier = + UserEntity.builder() + .withId(userId) + .withName(userName) + .withAuditInfo(auditInfo) + .withRolesSupplier( + () -> { + hasCallRolesSupplier.set(true); + return Collections.emptyList(); + }) + .build(); + + Assertions.assertFalse(hasCallRolesSupplier.get()); + + userWithSupplier.roles(); + Assertions.assertTrue(hasCallRolesSupplier.get()); } @Test @@ -258,19 +292,48 @@ public void testGroup() { GroupEntity.builder() .withId(groupId) .withName(groupName) + .withRoles( + Lists.newArrayList( + RoleEntity.builder() + .withId(1L) + .withName("role") + .withAuditInfo(auditInfo) + .build())) .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) .build(); Map fields = group.fields(); Assertions.assertEquals(groupId, fields.get(GroupEntity.ID)); Assertions.assertEquals(groupName, fields.get(GroupEntity.NAME)); Assertions.assertEquals(auditInfo, fields.get(GroupEntity.AUDIT_INFO)); - Assertions.assertEquals(Lists.newArrayList("role"), fields.get(GroupEntity.ROLE_NAMES)); + Assertions.assertEquals( + Lists.newArrayList( + RoleEntity.builder().withId(1L).withName("role").withAuditInfo(auditInfo).build()), + ((Supplier>) fields.get(GroupEntity.ROLES_SUPPLIER)).get()); GroupEntity groupWithoutFields = GroupEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); - Assertions.assertNull(groupWithoutFields.roles()); + Assertions.assertTrue(groupWithoutFields.roles().isEmpty()); + + AtomicBoolean hasCallRolesSupplier = new AtomicBoolean(false); + + // Test lazily loading + GroupEntity groupWithSupplier = + GroupEntity.builder() + .withId(userId) + .withName(userName) + .withAuditInfo(auditInfo) + .withRolesSupplier( + () -> { + hasCallRolesSupplier.set(true); + return Collections.emptyList(); + }) + .build(); + + Assertions.assertFalse(hasCallRolesSupplier.get()); + + groupWithSupplier.roles(); + Assertions.assertTrue(hasCallRolesSupplier.get()); } @Test @@ -296,7 +359,8 @@ public void testRole() { Lists.newArrayList( SecurableObjects.ofCatalog( catalogName, Lists.newArrayList(Privileges.UseCatalog.allow()))), - fields.get(RoleEntity.SECURABLE_OBJECT)); + ((Supplier>) fields.get(RoleEntity.SECURABLE_OBJECTS_SUPPLIER)) + .get()); RoleEntity roleWithoutFields = RoleEntity.builder() @@ -309,6 +373,26 @@ public void testRole() { catalogName, Lists.newArrayList(Privileges.UseCatalog.allow())))) .build(); Assertions.assertNull(roleWithoutFields.properties()); + + // Test lazily loading + AtomicBoolean hasCallSecurableObjectsSupplier = new AtomicBoolean(false); + + RoleEntity roleWithSupplier = + RoleEntity.builder() + .withId(userId) + .withName(userName) + .withAuditInfo(auditInfo) + .withSecurableObjectsSupplier( + () -> { + hasCallSecurableObjectsSupplier.set(true); + return null; + }) + .build(); + + Assertions.assertFalse(hasCallSecurableObjectsSupplier.get()); + + roleWithSupplier.securableObjects(); + Assertions.assertTrue(hasCallSecurableObjectsSupplier.get()); } @Test diff --git a/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java index 1d4bfaa17d9..226199a0d93 100644 --- a/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/org/apache/gravitino/proto/TestEntityProtoSerDe.java @@ -19,23 +19,15 @@ package org.apache.gravitino.proto; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import java.io.IOException; import java.time.Instant; import java.util.Map; import org.apache.gravitino.Catalog; -import org.apache.gravitino.Entity; import org.apache.gravitino.EntitySerDe; import org.apache.gravitino.EntitySerDeFactory; import org.apache.gravitino.Namespace; -import org.apache.gravitino.authorization.Privileges; -import org.apache.gravitino.authorization.SecurableObject; -import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.meta.GroupEntity; -import org.apache.gravitino.meta.RoleEntity; import org.apache.gravitino.meta.SchemaVersion; -import org.apache.gravitino.meta.UserEntity; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -323,110 +315,5 @@ public void testEntitiesSerDe() throws IOException { Assertions.assertEquals(topicEntity1, topicEntityFromBytes1); Assertions.assertNull(topicEntityFromBytes1.comment()); Assertions.assertNull(topicEntityFromBytes1.properties()); - - // Test UserEntity - Namespace userNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME); - Long userId = 1L; - String userName = "user"; - UserEntity userEntity = - UserEntity.builder() - .withId(userId) - .withName(userName) - .withNamespace(userNamespace) - .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) - .withRoleIds(Lists.newArrayList(1L)) - .build(); - byte[] userBytes = protoEntitySerDe.serialize(userEntity); - UserEntity userEntityFromBytes = - protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); - Assertions.assertEquals(userEntity, userEntityFromBytes); - - UserEntity userEntityWithoutFields = - UserEntity.builder() - .withId(userId) - .withName(userName) - .withNamespace(userNamespace) - .withAuditInfo(auditInfo) - .build(); - userBytes = protoEntitySerDe.serialize(userEntityWithoutFields); - userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); - Assertions.assertEquals(userEntityWithoutFields, userEntityFromBytes); - Assertions.assertNull(userEntityWithoutFields.roles()); - Assertions.assertNull(userEntityWithoutFields.roleIds()); - - // Test GroupEntity - Namespace groupNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME); - Long groupId = 1L; - String groupName = "group"; - - GroupEntity group = - GroupEntity.builder() - .withId(groupId) - .withName(groupName) - .withNamespace(groupNamespace) - .withAuditInfo(auditInfo) - .withRoleNames(Lists.newArrayList("role")) - .withRoleIds(Lists.newArrayList(1L)) - .build(); - byte[] groupBytes = protoEntitySerDe.serialize(group); - GroupEntity groupFromBytes = - protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); - Assertions.assertEquals(group, groupFromBytes); - - GroupEntity groupWithoutFields = - GroupEntity.builder() - .withId(groupId) - .withName(groupName) - .withNamespace(groupNamespace) - .withAuditInfo(auditInfo) - .build(); - groupBytes = protoEntitySerDe.serialize(groupWithoutFields); - groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); - Assertions.assertEquals(groupWithoutFields, groupFromBytes); - Assertions.assertNull(groupWithoutFields.roles()); - Assertions.assertNull(groupWithoutFields.roleIds()); - - // Test RoleEntity - Namespace roleNamespace = - Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME); - Long roleId = 1L; - String roleName = "testRole"; - String anotherCatalogName = "anotherCatalog"; - SecurableObject securableObject = - SecurableObjects.ofCatalog( - catalogName, - Lists.newArrayList(Privileges.UseCatalog.allow(), Privileges.CreateSchema.deny())); - SecurableObject anotherSecurableObject = - SecurableObjects.ofCatalog( - anotherCatalogName, Lists.newArrayList(Privileges.UseCatalog.allow())); - - RoleEntity roleEntity = - RoleEntity.builder() - .withId(roleId) - .withName(roleName) - .withNamespace(roleNamespace) - .withAuditInfo(auditInfo) - .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) - .withProperties(props) - .build(); - byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); - RoleEntity roleFromBytes = - protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); - Assertions.assertEquals(roleEntity, roleFromBytes); - - RoleEntity roleWithoutFields = - RoleEntity.builder() - .withId(1L) - .withName(roleName) - .withNamespace(roleNamespace) - .withAuditInfo(auditInfo) - .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) - .build(); - roleBytes = protoEntitySerDe.serialize(roleWithoutFields); - roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); - Assertions.assertEquals(roleWithoutFields, roleFromBytes); } } diff --git a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java index bdf95ff9639..d233ac5bafc 100644 --- a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java @@ -46,6 +46,7 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.UUID; import org.apache.commons.io.FileUtils; @@ -1234,7 +1235,7 @@ private static UserEntity createUser(Long id, String metalake, String name, Audi .withNamespace(AuthorizationUtils.ofUserNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); } @@ -1245,7 +1246,7 @@ private static GroupEntity createGroup( .withNamespace(AuthorizationUtils.ofGroupNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); } diff --git a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java index 7cff7bcc553..4d41d2f2523 100644 --- a/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java +++ b/core/src/test/java/org/apache/gravitino/storage/memory/TestMemoryEntityStore.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.locks.Lock; @@ -224,7 +225,7 @@ public void testEntityStoreAndRetrieve() throws Exception { .withName("user") .withNamespace(Namespace.of("metalake", "catalog", "db")) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); GroupEntity groupEntity = @@ -233,7 +234,7 @@ public void testEntityStoreAndRetrieve() throws Exception { .withName("group") .withNamespace(AuthorizationUtils.ofGroupNamespace("metalake")) .withAuditInfo(auditInfo) - .withRoleNames(null) + .withRoles(Collections.emptyList()) .build(); RoleEntity roleEntity = diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java index 26430d2fb5e..d72f542fa1f 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/TestJDBCBackend.java @@ -45,6 +45,7 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -600,8 +601,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofUserNamespace("metalake"), "user", auditInfo, - Lists.newArrayList(role.name()), - Lists.newArrayList(role.id())); + Lists.newArrayList(role)); backend.insert(user, false); GroupEntity group = @@ -610,8 +610,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofGroupNamespace("metalake"), "group", auditInfo, - Lists.newArrayList(role.name()), - Lists.newArrayList(role.id())); + Lists.newArrayList(role)); backend.insert(group, false); TagEntity tag = @@ -688,8 +687,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofUserNamespace("another-metalake"), "another-user", auditInfo, - Lists.newArrayList(anotherRole.name()), - Lists.newArrayList(anotherRole.id())); + Lists.newArrayList(anotherRole)); backend.insert(anotherUser, false); GroupEntity anotherGroup = @@ -698,8 +696,7 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { AuthorizationUtils.ofGroupNamespace("another-metalake"), "another-group", auditInfo, - Lists.newArrayList(anotherRole.name()), - Lists.newArrayList(anotherRole.id())); + Lists.newArrayList(anotherRole)); backend.insert(anotherGroup, false); TagEntity anotherTagEntity = @@ -1112,8 +1109,7 @@ public static UserEntity createUserEntity( .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(null) - .withRoleIds(null) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); } @@ -1123,14 +1119,12 @@ public static UserEntity createUserEntity( Namespace namespace, String name, AuditInfo auditInfo, - List roleNames, - List roleIds) { + List roleEntities) { return UserEntity.builder() .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(auditInfo) .build(); } @@ -1156,25 +1150,18 @@ public static GroupEntity createGroupEntity( .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(null) - .withRoleIds(null) + .withRoles(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); } public static GroupEntity createGroupEntity( - Long id, - Namespace namespace, - String name, - AuditInfo auditInfo, - List roleNames, - List roleIds) { + Long id, Namespace namespace, String name, AuditInfo auditInfo, List roles) { return GroupEntity.builder() .withId(id) .withName(name) .withNamespace(namespace) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roles) .withAuditInfo(auditInfo) .build(); } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java index 22246ba0cf3..d6b33095045 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestGroupMetaService.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.Namespace; @@ -110,13 +111,11 @@ void getGroupByIdentifier() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group2, false); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); Assertions.assertEquals(group2.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group2.roles()), Sets.newHashSet(actualGroup.roles())); } @Test @@ -194,13 +193,11 @@ void insertGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group2, false)); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); Assertions.assertEquals(group2.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group2.roles()), Sets.newHashSet(actualGroup.roles())); // insert duplicate group with roles GroupEntity group2Exist = @@ -219,17 +216,16 @@ void insertGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2Overwrite", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group2Overwrite, true)); GroupEntity actualOverwriteGroup2 = groupMetaService.getGroupByIdentifier(group2Overwrite.nameIdentifier()); Assertions.assertEquals("group2Overwrite", actualOverwriteGroup2.name()); - Assertions.assertEquals(2, actualOverwriteGroup2.roleNames().size()); + Assertions.assertEquals(2, actualOverwriteGroup2.roles().size()); Assertions.assertEquals( Sets.newHashSet(role1.name(), role2.name()), - Sets.newHashSet(actualOverwriteGroup2.roleNames())); + Sets.newHashSet(actualOverwriteGroup2.roles())); // insert overwrite user with 1 role RoleEntity role3 = @@ -246,15 +242,14 @@ void insertGroup() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "group3Overwrite", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); Assertions.assertDoesNotThrow(() -> groupMetaService.insertGroup(group3Overwrite, true)); GroupEntity actualOverwriteGroup3 = groupMetaService.getGroupByIdentifier(group3Overwrite.nameIdentifier()); Assertions.assertEquals("group3Overwrite", actualOverwriteGroup3.name()); - Assertions.assertEquals(1, actualOverwriteGroup3.roleNames().size()); - Assertions.assertEquals("role3", actualOverwriteGroup3.roleNames().get(0)); + Assertions.assertEquals(1, actualOverwriteGroup3.roles().size()); + Assertions.assertEquals("role3", actualOverwriteGroup3.roles().get(0)); // insert overwrite user with 0 roles GroupEntity group4Overwrite = @@ -268,7 +263,7 @@ void insertGroup() throws IOException { GroupEntity actualOverwriteGroup4 = groupMetaService.getGroupByIdentifier(group4Overwrite.nameIdentifier()); Assertions.assertEquals("group4Overwrite", actualOverwriteGroup4.name()); - Assertions.assertNull(actualOverwriteGroup4.roleNames()); + Assertions.assertTrue(actualOverwriteGroup4.roles().isEmpty()); } @Test @@ -341,8 +336,7 @@ void deleteGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group2, false); List rolePOs = SessionUtils.doWithCommitAndFetchResult( @@ -350,8 +344,7 @@ void deleteGroup() throws IOException { Assertions.assertEquals(2, rolePOs.size()); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group2.nameIdentifier()); Assertions.assertEquals(group2.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group2.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group2.roles()), Sets.newHashSet(actualGroup.roles())); Assertions.assertTrue(groupMetaService.deleteGroup(group2.nameIdentifier())); Assertions.assertThrows( @@ -400,13 +393,11 @@ void updateGroup() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group1, false); GroupEntity actualGroup = groupMetaService.getGroupByIdentifier(group1.nameIdentifier()); Assertions.assertEquals(group1.name(), actualGroup.name()); - Assertions.assertEquals( - Sets.newHashSet(group1.roleNames()), Sets.newHashSet(actualGroup.roleNames())); + Assertions.assertEquals(Sets.newHashSet(group1.roles()), Sets.newHashSet(actualGroup.roles())); RoleEntity role3 = createRoleEntity( @@ -428,17 +419,14 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); - roleNames.add(role3.name()); - roleIds.add(role3.id()); + List roleEntities = Lists.newArrayList(group.roleEntities()); + roleEntities.add(role3); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -449,9 +437,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), grantGroup.id()); Assertions.assertEquals(group1.name(), grantGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantGroup.roleNames())); + Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role2.id(), role3.id()), Sets.newHashSet(grantGroup.roleIds())); + Sets.newHashSet(role1.id(), role2.id(), role3.id()), + grantGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantGroup.auditInfo().creator()); Assertions.assertEquals("grantGroup", grantGroup.auditInfo().lastModifier()); @@ -466,17 +455,16 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); - roleIds.remove(roleNames.indexOf("role2")); - roleNames.remove("role2"); + List roleEntities = Lists.newArrayList(group.roleEntities()); + List roleNames = + roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList()); + roleEntities.remove(roleNames.indexOf("role2")); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -487,9 +475,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), revokeGroup.id()); Assertions.assertEquals(group1.name(), revokeGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeGroup.roleNames())); + Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role3.id()), Sets.newHashSet(revokeGroup.roleIds())); + Sets.newHashSet(role1.id(), role3.id()), + revokeGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", revokeGroup.auditInfo().creator()); Assertions.assertEquals("revokeGroup", revokeGroup.auditInfo().lastModifier()); @@ -513,19 +502,16 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); - roleIds.remove(roleNames.indexOf("role3")); - roleNames.remove("role3"); - roleIds.add(role4.id()); - roleNames.add(role4.name()); + List roleNames = Lists.newArrayList(group.roles()); + List roleEntities = Lists.newArrayList(group.roleEntities()); + roleEntities.remove(roleNames.indexOf("role3")); + roleEntities.add(role4); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -536,9 +522,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), grantRevokeGroup.id()); Assertions.assertEquals(group1.name(), grantRevokeGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeGroup.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(grantRevokeGroup.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + grantRevokeGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantRevokeGroup.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", grantRevokeGroup.auditInfo().lastModifier()); @@ -553,15 +540,13 @@ void updateGroup() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(group.roleNames()); - List roleIds = Lists.newArrayList(group.roleIds()); + List roleEntities = Lists.newArrayList(group.roleEntities()); return GroupEntity.builder() .withNamespace(group.namespace()) .withId(group.id()) .withName(group.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -571,9 +556,10 @@ void updateGroup() throws IOException { Assertions.assertEquals(group1.id(), noUpdaterGroup.id()); Assertions.assertEquals(group1.name(), noUpdaterGroup.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterGroup.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterGroup.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterGroup.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + noUpdaterGroup.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", noUpdaterGroup.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", noUpdaterGroup.auditInfo().lastModifier()); @@ -581,7 +567,7 @@ void updateGroup() throws IOException { RoleMetaService.getInstance().deleteRole(role1.nameIdentifier()); GroupEntity groupEntity = GroupMetaService.getInstance().getGroupByIdentifier(group1.nameIdentifier()); - Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(groupEntity.roleNames())); + Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(groupEntity.roles())); } @Test @@ -629,16 +615,14 @@ void deleteMetalake() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); @@ -710,16 +694,14 @@ void deleteMetalakeCascade() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); @@ -782,32 +764,28 @@ void deleteGroupMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group3 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "group3", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); GroupEntity group4 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "group4", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); groupMetaService.insertGroup(group3, false); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java index d811b8b598f..1db24de1c9b 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestRoleMetaService.java @@ -301,41 +301,36 @@ void deleteRole() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role2.name()), - Lists.newArrayList(role2.id())); + Lists.newArrayList(role2)); GroupEntity group2 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofGroupNamespace(metalakeName), "group2", auditInfo, - Lists.newArrayList(role2.name()), - Lists.newArrayList(role2.id())); + Lists.newArrayList(role2)); groupMetaService.insertGroup(group1, false); groupMetaService.insertGroup(group2, false); Assertions.assertEquals( group1.name(), groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).name()); Assertions.assertEquals( - group1.roleNames(), - groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).roleNames()); + group1.roles(), groupMetaService.getGroupByIdentifier(group1.nameIdentifier()).roles()); Assertions.assertEquals( group2.name(), groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).name()); Assertions.assertEquals( - group2.roleNames(), - groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).roleNames()); + group2.roles(), groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).roles()); UserEntity user1 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role2.name()), - Lists.newArrayList(role2.id())); + Lists.newArrayList(role2)); userMetaService.insertUser(user1, false); Assertions.assertEquals( user1.name(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).name()); Assertions.assertEquals( - user1.roleNames(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).roleNames()); + user1.roles(), userMetaService.getUserByIdentifier(user1.nameIdentifier()).roles()); Assertions.assertTrue(roleMetaService.deleteRole(role2.nameIdentifier())); Assertions.assertThrows( @@ -434,16 +429,14 @@ void deleteMetalake() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user1 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); groupMetaService.insertGroup(group1, false); @@ -546,16 +539,14 @@ void deleteMetalakeCascade() throws IOException { AuthorizationUtils.ofGroupNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user1 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); groupMetaService.insertGroup(group1, false); @@ -658,8 +649,7 @@ void deleteRoleMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user1, false); GroupEntity group1 = @@ -668,8 +658,7 @@ void deleteRoleMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "group1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); groupMetaService.insertGroup(group1, false); // hard delete before soft delete diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java index 326ccfc2da3..90e675e06df 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestUserMetaService.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.Namespace; @@ -109,13 +110,11 @@ void getUserByIdentifier() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user2, false); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user2.roles()), Sets.newHashSet(actualUser.roles())); } @Test @@ -193,13 +192,11 @@ void insertUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user2, false)); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user2.roles()), Sets.newHashSet(actualUser.roles())); // insert duplicate user with roles UserEntity user2Exist = @@ -218,17 +215,15 @@ void insertUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2Overwrite", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user2Overwrite, true)); UserEntity actualOverwriteUser2 = userMetaService.getUserByIdentifier(user2Overwrite.nameIdentifier()); Assertions.assertEquals("user2Overwrite", actualOverwriteUser2.name()); - Assertions.assertEquals(2, actualOverwriteUser2.roleNames().size()); + Assertions.assertEquals(2, actualOverwriteUser2.roles().size()); Assertions.assertEquals( - Sets.newHashSet(role1.name(), role2.name()), - Sets.newHashSet(actualOverwriteUser2.roleNames())); + Sets.newHashSet(role1.name(), role2.name()), Sets.newHashSet(actualOverwriteUser2.roles())); // insert overwrite user with 1 role RoleEntity role3 = @@ -245,15 +240,14 @@ void insertUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user3Overwrite", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); Assertions.assertDoesNotThrow(() -> userMetaService.insertUser(user3Overwrite, true)); UserEntity actualOverwriteUser3 = userMetaService.getUserByIdentifier(user3Overwrite.nameIdentifier()); Assertions.assertEquals("user3Overwrite", actualOverwriteUser3.name()); - Assertions.assertEquals(1, actualOverwriteUser3.roleNames().size()); - Assertions.assertEquals("role3", actualOverwriteUser3.roleNames().get(0)); + Assertions.assertEquals(1, actualOverwriteUser3.roles().size()); + Assertions.assertEquals("role3", actualOverwriteUser3.roles().get(0)); // insert overwrite user with 0 roles UserEntity user4Overwrite = @@ -267,7 +261,7 @@ void insertUser() throws IOException { UserEntity actualOverwriteUser4 = userMetaService.getUserByIdentifier(user4Overwrite.nameIdentifier()); Assertions.assertEquals("user4Overwrite", actualOverwriteUser4.name()); - Assertions.assertNull(actualOverwriteUser4.roleNames()); + Assertions.assertTrue(actualOverwriteUser4.roles().isEmpty()); } @Test @@ -338,8 +332,7 @@ void deleteUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user2, false); List rolePOs = SessionUtils.doWithCommitAndFetchResult( @@ -347,8 +340,7 @@ void deleteUser() throws IOException { Assertions.assertEquals(2, rolePOs.size()); UserEntity actualUser = userMetaService.getUserByIdentifier(user2.nameIdentifier()); Assertions.assertEquals(user2.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user2.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user2.roles()), Sets.newHashSet(actualUser.roles())); Assertions.assertTrue(userMetaService.deleteUser(user2.nameIdentifier())); Assertions.assertThrows( @@ -397,13 +389,11 @@ void updateUser() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user1, false); UserEntity actualUser = userMetaService.getUserByIdentifier(user1.nameIdentifier()); Assertions.assertEquals(user1.name(), actualUser.name()); - Assertions.assertEquals( - Sets.newHashSet(user1.roleNames()), Sets.newHashSet(actualUser.roleNames())); + Assertions.assertEquals(Sets.newHashSet(user1.roles()), Sets.newHashSet(actualUser.roles())); RoleEntity role3 = createRoleEntity( @@ -425,17 +415,14 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); - roleNames.add(role3.name()); - roleIds.add(role3.id()); + List roleEntities = Lists.newArrayList(user.roleEntities()); + roleEntities.add(role3); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -446,9 +433,10 @@ void updateUser() throws IOException { Assertions.assertEquals(user1.id(), grantUser.id()); Assertions.assertEquals(user1.name(), grantUser.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantUser.roleNames())); + Sets.newHashSet("role1", "role2", "role3"), Sets.newHashSet(grantUser.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role2.id(), role3.id()), Sets.newHashSet(grantUser.roleIds())); + Sets.newHashSet(role1.id(), role2.id(), role3.id()), + grantUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantUser.auditInfo().creator()); Assertions.assertEquals("grantUser", grantUser.auditInfo().lastModifier()); @@ -463,17 +451,15 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); - roleIds.remove(roleNames.indexOf("role2")); - roleNames.remove("role2"); + List roleNames = Lists.newArrayList(user.roles()); + List roleEntities = Lists.newArrayList(user.roleEntities()); + roleEntities.remove(roleNames.indexOf("role2")); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -483,10 +469,10 @@ void updateUser() throws IOException { UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); Assertions.assertEquals(user1.id(), revokeUser.id()); Assertions.assertEquals(user1.name(), revokeUser.name()); + Assertions.assertEquals(Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeUser.roles())); Assertions.assertEquals( - Sets.newHashSet("role1", "role3"), Sets.newHashSet(revokeUser.roleNames())); - Assertions.assertEquals( - Sets.newHashSet(role1.id(), role3.id()), Sets.newHashSet(revokeUser.roleIds())); + Sets.newHashSet(role1.id(), role3.id()), + revokeUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", revokeUser.auditInfo().creator()); Assertions.assertEquals("revokeUser", revokeUser.auditInfo().lastModifier()); @@ -510,19 +496,16 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); - roleIds.remove(roleNames.indexOf("role3")); - roleNames.remove("role3"); - roleIds.add(role4.id()); - roleNames.add(role4.name()); + List roleNames = Lists.newArrayList(user.roles()); + List roleEntities = Lists.newArrayList(user.roleEntities()); + roleEntities.remove(roleNames.indexOf("role3")); + roleEntities.add(role4); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -533,9 +516,10 @@ void updateUser() throws IOException { Assertions.assertEquals(user1.id(), grantRevokeUser.id()); Assertions.assertEquals(user1.name(), grantRevokeUser.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeUser.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(grantRevokeUser.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(grantRevokeUser.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + grantRevokeUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", grantRevokeUser.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", grantRevokeUser.auditInfo().lastModifier()); @@ -549,15 +533,13 @@ void updateUser() throws IOException { .withLastModifiedTime(Instant.now()) .build(); - List roleNames = Lists.newArrayList(user.roleNames()); - List roleIds = Lists.newArrayList(user.roleIds()); + List roleEntities = Lists.newArrayList(user.roleEntities()); return UserEntity.builder() .withNamespace(user.namespace()) .withId(user.id()) .withName(user.name()) - .withRoleNames(roleNames) - .withRoleIds(roleIds) + .withRoles(roleEntities) .withAuditInfo(updateAuditInfo) .build(); }; @@ -567,9 +549,10 @@ void updateUser() throws IOException { Assertions.assertEquals(user1.id(), noUpdaterUser.id()); Assertions.assertEquals(user1.name(), noUpdaterUser.name()); Assertions.assertEquals( - Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterUser.roleNames())); + Sets.newHashSet("role1", "role4"), Sets.newHashSet(noUpdaterUser.roles())); Assertions.assertEquals( - Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterUser.roleIds())); + Sets.newHashSet(role1.id(), role4.id()), + noUpdaterUser.roleEntities().stream().map(RoleEntity::id).collect(Collectors.toSet())); Assertions.assertEquals("creator", noUpdaterUser.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", noUpdaterUser.auditInfo().lastModifier()); @@ -577,7 +560,7 @@ void updateUser() throws IOException { RoleMetaService.getInstance().deleteRole(role1.nameIdentifier()); UserEntity userEntity = UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); - Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(userEntity.roleNames())); + Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(userEntity.roles())); } @Test @@ -625,16 +608,14 @@ void deleteMetalake() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); @@ -706,16 +687,14 @@ void deleteMetalakeCascade() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role3.name()), - Lists.newArrayList(role3.id())); + Lists.newArrayList(role3)); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); @@ -778,32 +757,28 @@ void deleteUserMetasByLegacyTimeline() throws IOException { AuthorizationUtils.ofUserNamespace(metalakeName), "user1", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user2 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user2", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user3 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user3", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); UserEntity user4 = createUserEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofUserNamespace(metalakeName), "user4", auditInfo, - Lists.newArrayList(role1.name(), role2.name()), - Lists.newArrayList(role1.id(), role2.id())); + Lists.newArrayList(role1, role2)); userMetaService.insertUser(user1, false); userMetaService.insertUser(user2, false); userMetaService.insertUser(user3, false); diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 16b5f2525fa..60b0be907a5 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -17,11 +17,18 @@ * under the License. */ +import com.github.gradle.node.NodeExtension import com.github.gradle.node.npm.task.NpxTask +configure { + version = "21.6.1" + npmVersion = "10.2.4" + download = true +} + tasks { val lintOpenAPI by registering(NpxTask::class) { - command.set("@redocly/cli@1.5.0") + command.set("@redocly/cli@1.23.1") args.set(listOf("lint", "--extends=recommended-strict", "${project.projectDir}/open-api/openapi.yaml")) } diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 738f0e432bb..0fc3af157c0 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -284,16 +284,21 @@ You can pass [Iceberg table properties](https://iceberg.apache.org/docs/1.5.2/co The Gravitino server doesn't allow passing the following reserved fields. -| Configuration item | Description | -|---------------------------------|---------------------------------------------------------| -| `comment` | The table comment. | -| `creator` | The table creator. | -| `location` | Iceberg location for table storage. | -| `current-snapshot-id` | The snapshot represents the current state of the table. | -| `cherry-pick-snapshot-id` | Selecting a specific snapshot in a merge operation. | -| `sort-order` | Selecting a specific snapshot in a merge operation. | -| `identifier-fields` | The identifier fields for defining the table. | -| `write.distribution-mode` | Defines distribution of write data | +| Configuration item | Description | Since Version | +|---------------------------|--------------------------------------------------------------------------------------|---------------| +| `comment` | The table comment, please use `comment` field in table meta instead. | 0.2.0 | +| `creator` | The table creator. | 0.2.0 | +| `current-snapshot-id` | The snapshot represents the current state of the table. | 0.2.0 | +| `cherry-pick-snapshot-id` | Selecting a specific snapshot in a merge operation. | 0.2.0 | +| `sort-order` | Iceberg table sort order, please use `SortOrder` in table meta instead. | 0.2.0 | +| `identifier-fields` | The identifier fields for defining the table. | 0.2.0 | +| `write.distribution-mode` | Defines distribution of write data, please use `distribution` in table meta instead. | 0.2.0 | + +Gravitino server doesn't allow to change such properties: + +| Configuration item | Description | Default value | Required | Since Version | +|--------------------|----------------------------------------------|---------------|----------|---------------| +| `location` | Iceberg location for table storage. | None | No | 0.2.0 | ### Table indexes diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml index 9c2cc3ca911..f2b3fe83598 100644 --- a/docs/open-api/filesets.yaml +++ b/docs/open-api/filesets.yaml @@ -157,7 +157,7 @@ paths: "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" - /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/filesets/{fileset}/fileLocation: + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/filesets/{fileset}/location: parameters: - $ref: "./openapi.yaml#/components/parameters/metalake" - $ref: "./openapi.yaml#/components/parameters/catalog" @@ -166,12 +166,12 @@ paths: get: tags: - - fileLocation + - location summary: Get file location operationId: getFileLocation description: Returns the specified file location object parameters: - - name: subPath + - name: sub_path in: query required: true schema: diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java index b0ff97ea68d..6b2b147b1c8 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/PostgreSQLContainer.java @@ -116,7 +116,7 @@ public void createDatabase(TestDatabaseName testDatabaseName) { DriverManager.getConnection(getJdbcUrl(), getUsername(), getPassword()); Statement statement = connection.createStatement()) { - String query = format("CREATE DATABASE %s;", testDatabaseName); + String query = format("CREATE DATABASE \"%s\"", testDatabaseName); statement.execute(query); LOG.info(format("PostgreSQL container database %s has been created", testDatabaseName)); } catch (SQLException e) { diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java index 968eeeef884..3e264652b31 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/AbstractIT.java @@ -19,6 +19,8 @@ package org.apache.gravitino.integration.test.util; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH; +import static org.apache.gravitino.integration.test.util.TestDatabaseName.PG_CATALOG_POSTGRESQL_IT; +import static org.apache.gravitino.integration.test.util.TestDatabaseName.PG_JDBC_BACKEND; import static org.apache.gravitino.server.GravitinoServer.WEBSERVER_CONF_PREFIX; import com.google.common.base.Splitter; @@ -30,6 +32,7 @@ import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; @@ -39,6 +42,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; @@ -49,6 +53,7 @@ import org.apache.gravitino.integration.test.MiniGravitinoContext; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.MySQLContainer; +import org.apache.gravitino.integration.test.container.PostgreSQLContainer; import org.apache.gravitino.server.GravitinoServer; import org.apache.gravitino.server.ServerConfig; import org.apache.gravitino.server.web.JettyServerConfig; @@ -92,6 +97,7 @@ public class AbstractIT { private static TestDatabaseName META_DATA; private static MySQLContainer MYSQL_CONTAINER; + private static PostgreSQLContainer POSTGRESQL_CONTAINER; protected static String serverUri; @@ -155,6 +161,63 @@ protected static void downLoadJDBCDriver() throws IOException { } } + protected static void setPGBackend() throws SQLException { + String pgUrlWithoutSchema = POSTGRESQL_CONTAINER.getJdbcUrl(META_DATA); + customConfigs.put(Configs.ENTITY_STORE_KEY, "relational"); + customConfigs.put(Configs.ENTITY_RELATIONAL_STORE_KEY, "JDBCBackend"); + customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY, pgUrlWithoutSchema); + customConfigs.put( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER_KEY, + POSTGRESQL_CONTAINER.getDriverClassName(META_DATA)); + customConfigs.put( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER_KEY, POSTGRESQL_CONTAINER.getUsername()); + customConfigs.put( + Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD_KEY, POSTGRESQL_CONTAINER.getPassword()); + + LOG.info("PG URL: {}", pgUrlWithoutSchema); + + String randomSchemaName = RandomStringUtils.random(10, true, false); + // Connect to the PostgreSQL docker and create a schema + String currentExecuteSql = ""; + try (Connection connection = + DriverManager.getConnection( + pgUrlWithoutSchema, + POSTGRESQL_CONTAINER.getUsername(), + POSTGRESQL_CONTAINER.getPassword())) { + connection.setCatalog(PG_CATALOG_POSTGRESQL_IT.toString()); + final Statement statement = connection.createStatement(); + statement.execute("drop schema if exists " + randomSchemaName); + statement.execute("create schema " + randomSchemaName); + statement.execute("set search_path to " + randomSchemaName); + String gravitinoHome = System.getenv("GRAVITINO_ROOT_DIR"); + String mysqlContent = + FileUtils.readFileToString( + new File( + gravitinoHome + + String.format( + "/scripts/postgresql/schema-%s-postgresql.sql", + ConfigConstants.VERSION_0_7_0)), + "UTF-8"); + + String[] initPGBackendSqls = + Arrays.stream(mysqlContent.split(";")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toArray(String[]::new); + + for (String sql : initPGBackendSqls) { + currentExecuteSql = sql; + statement.execute(sql); + } + } catch (Exception e) { + LOG.error("Failed to create database in pg, sql:\n{}", currentExecuteSql, e); + throw new RuntimeException(e); + } + + pgUrlWithoutSchema = pgUrlWithoutSchema + "?currentSchema=" + randomSchemaName; + customConfigs.put(Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL_KEY, pgUrlWithoutSchema); + } + private static void setMySQLBackend() { String mysqlUrl = MYSQL_CONTAINER.getJdbcUrl(META_DATA); customConfigs.put(Configs.ENTITY_STORE_KEY, "relational"); @@ -221,6 +284,13 @@ public static void startIntegrationTest() throws Exception { MYSQL_CONTAINER = containerSuite.getMySQLContainer(); setMySQLBackend(); + } else if ("PostgreSQL".equalsIgnoreCase(System.getenv("jdbcBackend"))) { + // Start PostgreSQL docker instance. + META_DATA = PG_JDBC_BACKEND; + containerSuite.startPostgreSQLContainer(META_DATA); + POSTGRESQL_CONTAINER = containerSuite.getPostgreSQLContainer(); + + setPGBackend(); } File baseDir = new File(System.getProperty("java.io.tmpdir")); diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java index 98fb959ae77..4e81b992cc3 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/TestDatabaseName.java @@ -58,6 +58,8 @@ public enum TestDatabaseName { /** Represents the MySQL database used for testing the catalog integration with MySQL. */ MYSQL_CATALOG_MYSQL_IT, + PG_JDBC_BACKEND, + /** Represents the PostgreSQL database for CatalogPostgreSqlIT. */ PG_CATALOG_POSTGRESQL_IT { /** PostgreSQL only accept lowercase database name */ diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java deleted file mode 100644 index db189d525a9..00000000000 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.integration.test.client; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.google.common.collect.Maps; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.apache.gravitino.Catalog; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.audit.CallerContext; -import org.apache.gravitino.audit.FilesetAuditConstants; -import org.apache.gravitino.client.GravitinoMetalake; -import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.integration.test.container.ContainerSuite; -import org.apache.gravitino.integration.test.container.HiveContainer; -import org.apache.gravitino.integration.test.util.AbstractIT; -import org.apache.gravitino.integration.test.util.GravitinoITUtils; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Tag("gravitino-docker-test") -public class FilesetIT extends AbstractIT { - private static final Logger LOG = LoggerFactory.getLogger(FilesetIT.class); - - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - - private static final String metalakeName = GravitinoITUtils.genRandomName("fileset_it_metalake"); - - private static GravitinoMetalake metalake; - - private static String hmsUri; - - @BeforeAll - public static void startUp() { - containerSuite.startHiveContainer(); - hmsUri = - String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT); - - Assertions.assertFalse(client.metalakeExists(metalakeName)); - metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); - Assertions.assertTrue(client.metalakeExists(metalakeName)); - } - - @AfterAll - public static void tearDown() { - client.dropMetalake(metalakeName); - - if (client != null) { - client.close(); - client = null; - } - - try { - closer.close(); - } catch (Exception e) { - LOG.error("Exception in closing CloseableGroup", e); - } - } - - @AfterEach - public void cleanUp() { - String[] catalog = metalake.listCatalogs(); - for (String catalogName : catalog) { - Catalog filesetCatalog = metalake.loadCatalog(catalogName); - String[] schemas = filesetCatalog.asSchemas().listSchemas(); - for (String schemaName : schemas) { - filesetCatalog.asSchemas().dropSchema(schemaName, true); - } - metalake.dropCatalog(catalogName); - } - } - - @Test - public void testGetFileLocation() { - String catalogName = GravitinoITUtils.genRandomName("catalog"); - Assertions.assertFalse(metalake.catalogExists(catalogName)); - - Map properties = Maps.newHashMap(); - properties.put("metastore.uris", hmsUri); - Catalog catalog = - metalake.createCatalog( - catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); - Assertions.assertTrue(metalake.catalogExists(catalogName)); - - String schemaName = GravitinoITUtils.genRandomName("schema"); - Assertions.assertFalse(catalog.asSchemas().schemaExists(schemaName)); - catalog.asSchemas().createSchema(schemaName, "schema comment", Maps.newHashMap()); - Assertions.assertTrue(catalog.asSchemas().schemaExists(schemaName)); - - String filesetName = GravitinoITUtils.genRandomName("fileset"); - NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); - Assertions.assertFalse(catalog.asFilesetCatalog().filesetExists(filesetIdent)); - Fileset expectedFileset = - catalog - .asFilesetCatalog() - .createFileset( - filesetIdent, - "fileset comment", - Fileset.Type.MANAGED, - generateLocation(catalogName, schemaName, filesetName), - Maps.newHashMap()); - Assertions.assertTrue(catalog.asFilesetCatalog().filesetExists(filesetIdent)); - - String actualFileLocation = - catalog.asFilesetCatalog().getFileLocation(filesetIdent, "/test.par"); - - Assertions.assertEquals(expectedFileset.storageLocation() + "/test.par", actualFileLocation); - } - - @Test - public void testGetFileLocationWithInvalidAuditHeaders() { - try { - String catalogName = GravitinoITUtils.genRandomName("catalog"); - String schemaName = GravitinoITUtils.genRandomName("schema"); - String filesetName = GravitinoITUtils.genRandomName("fileset"); - NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); - - Map properties = Maps.newHashMap(); - properties.put("metastore.uris", hmsUri); - Catalog catalog = - metalake.createCatalog( - catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); - Assertions.assertTrue(metalake.catalogExists(catalogName)); - - Map context = new HashMap<>(); - // this is a invalid internal client type. - context.put(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, "test"); - CallerContext callerContext = CallerContext.builder().withContext(context).build(); - CallerContext.CallerContextHolder.set(callerContext); - - assertThrows( - IllegalArgumentException.class, - () -> catalog.asFilesetCatalog().getFileLocation(filesetIdent, "/test.par")); - } finally { - CallerContext.CallerContextHolder.remove(); - } - } - - private static String generateLocation( - String catalogName, String schemaName, String filesetName) { - return String.format( - "hdfs://%s:%d/user/hadoop/%s/%s/%s", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT, - catalogName, - schemaName, - filesetName); - } -} diff --git a/scripts/postgresql/schema-0.7.0-postgresql.sql b/scripts/postgresql/schema-0.7.0-postgresql.sql new file mode 100644 index 00000000000..0ce7a968537 --- /dev/null +++ b/scripts/postgresql/schema-0.7.0-postgresql.sql @@ -0,0 +1,458 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +-- Note: Database and schema creation is not included in this script. Please create the database and +-- schema before running this script. for example in psql: +-- CREATE DATABASE example_db; +-- \c example_db +-- CREATE SCHEMA example_schema; +-- set search_path to example_schema; + + +CREATE TABLE IF NOT EXISTS metalake_meta ( + metalake_id BIGINT NOT NULL, + metalake_name VARCHAR(128) NOT NULL, + metalake_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + schema_version TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (metalake_id), + UNIQUE (metalake_name, deleted_at) + ); +COMMENT ON TABLE metalake_meta IS 'metalake metadata'; + +COMMENT ON COLUMN metalake_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN metalake_meta.metalake_name IS 'metalake name'; +COMMENT ON COLUMN metalake_meta.metalake_comment IS 'metalake comment'; +COMMENT ON COLUMN metalake_meta.properties IS 'metalake properties'; +COMMENT ON COLUMN metalake_meta.audit_info IS 'metalake audit info'; +COMMENT ON COLUMN metalake_meta.schema_version IS 'metalake schema version info'; +COMMENT ON COLUMN metalake_meta.current_version IS 'metalake current version'; +COMMENT ON COLUMN metalake_meta.last_version IS 'metalake last version'; +COMMENT ON COLUMN metalake_meta.deleted_at IS 'metalake deleted at'; + + +CREATE TABLE IF NOT EXISTS catalog_meta ( + catalog_id BIGINT NOT NULL, + catalog_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + type VARCHAR(64) NOT NULL, + provider VARCHAR(64) NOT NULL, + catalog_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (catalog_id), + UNIQUE (metalake_id, catalog_name, deleted_at) + ); + +COMMENT ON TABLE catalog_meta IS 'catalog metadata'; + +COMMENT ON COLUMN catalog_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN catalog_meta.catalog_name IS 'catalog name'; +COMMENT ON COLUMN catalog_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN catalog_meta.type IS 'catalog type'; +COMMENT ON COLUMN catalog_meta.provider IS 'catalog provider'; +COMMENT ON COLUMN catalog_meta.catalog_comment IS 'catalog comment'; +COMMENT ON COLUMN catalog_meta.properties IS 'catalog properties'; +COMMENT ON COLUMN catalog_meta.audit_info IS 'catalog audit info'; +COMMENT ON COLUMN catalog_meta.current_version IS 'catalog current version'; +COMMENT ON COLUMN catalog_meta.last_version IS 'catalog last version'; +COMMENT ON COLUMN catalog_meta.deleted_at IS 'catalog deleted at'; + + +CREATE TABLE IF NOT EXISTS schema_meta ( + schema_id BIGINT NOT NULL, + schema_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (schema_id), + UNIQUE (catalog_id, schema_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON schema_meta (metalake_id); +COMMENT ON TABLE schema_meta IS 'schema metadata'; + +COMMENT ON COLUMN schema_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN schema_meta.schema_name IS 'schema name'; +COMMENT ON COLUMN schema_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN schema_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN schema_meta.schema_comment IS 'schema comment'; +COMMENT ON COLUMN schema_meta.properties IS 'schema properties'; +COMMENT ON COLUMN schema_meta.audit_info IS 'schema audit info'; +COMMENT ON COLUMN schema_meta.current_version IS 'schema current version'; +COMMENT ON COLUMN schema_meta.last_version IS 'schema last version'; +COMMENT ON COLUMN schema_meta.deleted_at IS 'schema deleted at'; + + +CREATE TABLE IF NOT EXISTS table_meta ( + table_id BIGINT NOT NULL, + table_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (table_id), + UNIQUE (schema_id, table_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON table_meta (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON table_meta (catalog_id); +COMMENT ON TABLE table_meta IS 'table metadata'; + +COMMENT ON COLUMN table_meta.table_id IS 'table id'; +COMMENT ON COLUMN table_meta.table_name IS 'table name'; +COMMENT ON COLUMN table_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN table_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN table_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN table_meta.audit_info IS 'table audit info'; +COMMENT ON COLUMN table_meta.current_version IS 'table current version'; +COMMENT ON COLUMN table_meta.last_version IS 'table last version'; +COMMENT ON COLUMN table_meta.deleted_at IS 'table deleted at'; + + +CREATE TABLE IF NOT EXISTS fileset_meta ( + fileset_id BIGINT NOT NULL, + fileset_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + type VARCHAR(64) NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (fileset_id), + UNIQUE (schema_id, fileset_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON fileset_meta (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON fileset_meta (catalog_id); +COMMENT ON TABLE fileset_meta IS 'fileset metadata'; + +COMMENT ON COLUMN fileset_meta.fileset_id IS 'fileset id'; +COMMENT ON COLUMN fileset_meta.fileset_name IS 'fileset name'; +COMMENT ON COLUMN fileset_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN fileset_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN fileset_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN fileset_meta.type IS 'fileset type'; +COMMENT ON COLUMN fileset_meta.audit_info IS 'fileset audit info'; +COMMENT ON COLUMN fileset_meta.current_version IS 'fileset current version'; +COMMENT ON COLUMN fileset_meta.last_version IS 'fileset last version'; +COMMENT ON COLUMN fileset_meta.deleted_at IS 'fileset deleted at'; + + +CREATE TABLE IF NOT EXISTS fileset_version_info ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + fileset_id BIGINT NOT NULL, + version INT NOT NULL, + fileset_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + storage_location TEXT NOT NULL, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (fileset_id, version, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON fileset_version_info (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON fileset_version_info (catalog_id); +CREATE INDEX IF NOT EXISTS idx_schema_id ON fileset_version_info (schema_id); +COMMENT ON TABLE fileset_version_info IS 'fileset version information'; + +COMMENT ON COLUMN fileset_version_info.id IS 'auto increment id'; +COMMENT ON COLUMN fileset_version_info.metalake_id IS 'metalake id'; +COMMENT ON COLUMN fileset_version_info.catalog_id IS 'catalog id'; +COMMENT ON COLUMN fileset_version_info.schema_id IS 'schema id'; +COMMENT ON COLUMN fileset_version_info.fileset_id IS 'fileset id'; +COMMENT ON COLUMN fileset_version_info.version IS 'fileset info version'; +COMMENT ON COLUMN fileset_version_info.fileset_comment IS 'fileset comment'; +COMMENT ON COLUMN fileset_version_info.properties IS 'fileset properties'; +COMMENT ON COLUMN fileset_version_info.storage_location IS 'fileset storage location'; +COMMENT ON COLUMN fileset_version_info.deleted_at IS 'fileset deleted at'; + + +CREATE TABLE IF NOT EXISTS topic_meta ( + topic_id BIGINT NOT NULL, + topic_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + catalog_id BIGINT NOT NULL, + schema_id BIGINT NOT NULL, + comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (topic_id), + UNIQUE (schema_id, topic_name, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_metalake_id ON topic_meta (metalake_id); +CREATE INDEX IF NOT EXISTS idx_catalog_id ON topic_meta (catalog_id); +COMMENT ON TABLE topic_meta IS 'topic metadata'; + +COMMENT ON COLUMN topic_meta.topic_id IS 'topic id'; +COMMENT ON COLUMN topic_meta.topic_name IS 'topic name'; +COMMENT ON COLUMN topic_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN topic_meta.catalog_id IS 'catalog id'; +COMMENT ON COLUMN topic_meta.schema_id IS 'schema id'; +COMMENT ON COLUMN topic_meta.comment IS 'topic comment'; +COMMENT ON COLUMN topic_meta.properties IS 'topic properties'; +COMMENT ON COLUMN topic_meta.audit_info IS 'topic audit info'; +COMMENT ON COLUMN topic_meta.current_version IS 'topic current version'; +COMMENT ON COLUMN topic_meta.last_version IS 'topic last version'; +COMMENT ON COLUMN topic_meta.deleted_at IS 'topic deleted at'; + + +CREATE TABLE IF NOT EXISTS user_meta ( + user_id BIGINT NOT NULL, + user_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (user_id), + UNIQUE (metalake_id, user_name, deleted_at) + ); +COMMENT ON TABLE user_meta IS 'user metadata'; + +COMMENT ON COLUMN user_meta.user_id IS 'user id'; +COMMENT ON COLUMN user_meta.user_name IS 'username'; +COMMENT ON COLUMN user_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN user_meta.audit_info IS 'user audit info'; +COMMENT ON COLUMN user_meta.current_version IS 'user current version'; +COMMENT ON COLUMN user_meta.last_version IS 'user last version'; +COMMENT ON COLUMN user_meta.deleted_at IS 'user deleted at'; + +CREATE TABLE IF NOT EXISTS role_meta ( + role_id BIGINT NOT NULL, + role_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (role_id), + UNIQUE (metalake_id, role_name, deleted_at) + ); + +COMMENT ON TABLE role_meta IS 'role metadata'; + +COMMENT ON COLUMN role_meta.role_id IS 'role id'; +COMMENT ON COLUMN role_meta.role_name IS 'role name'; +COMMENT ON COLUMN role_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN role_meta.properties IS 'role properties'; +COMMENT ON COLUMN role_meta.audit_info IS 'role audit info'; +COMMENT ON COLUMN role_meta.current_version IS 'role current version'; +COMMENT ON COLUMN role_meta.last_version IS 'role last version'; +COMMENT ON COLUMN role_meta.deleted_at IS 'role deleted at'; + + +CREATE TABLE IF NOT EXISTS role_meta_securable_object ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + role_id BIGINT NOT NULL, + metadata_object_id BIGINT NOT NULL, + type VARCHAR(128) NOT NULL, + privilege_names VARCHAR(256) NOT NULL, + privilege_conditions VARCHAR(256) NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id) + ); + +CREATE INDEX IF NOT EXISTS idx_role_id ON role_meta_securable_object (role_id); +COMMENT ON TABLE role_meta_securable_object IS 'role to securable object relation metadata'; + +COMMENT ON COLUMN role_meta_securable_object.id IS 'auto increment id'; +COMMENT ON COLUMN role_meta_securable_object.role_id IS 'role id'; +COMMENT ON COLUMN role_meta_securable_object.metadata_object_id IS 'The entity id of securable object'; +COMMENT ON COLUMN role_meta_securable_object.type IS 'securable object type'; +COMMENT ON COLUMN role_meta_securable_object.privilege_names IS 'securable object privilege names'; +COMMENT ON COLUMN role_meta_securable_object.privilege_conditions IS 'securable object privilege conditions'; +COMMENT ON COLUMN role_meta_securable_object.current_version IS 'securable object current version'; +COMMENT ON COLUMN role_meta_securable_object.last_version IS 'securable object last version'; +COMMENT ON COLUMN role_meta_securable_object.deleted_at IS 'securable object deleted at'; + + +CREATE TABLE IF NOT EXISTS user_role_rel ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + user_id BIGINT NOT NULL, + role_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (user_id, role_id, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_user_id ON user_role_rel (user_id); +COMMENT ON TABLE user_role_rel IS 'user role relation metadata'; + +COMMENT ON COLUMN user_role_rel.id IS 'auto increment id'; +COMMENT ON COLUMN user_role_rel.user_id IS 'user id'; +COMMENT ON COLUMN user_role_rel.role_id IS 'role id'; +COMMENT ON COLUMN user_role_rel.audit_info IS 'relation audit info'; +COMMENT ON COLUMN user_role_rel.current_version IS 'relation current version'; +COMMENT ON COLUMN user_role_rel.last_version IS 'relation last version'; +COMMENT ON COLUMN user_role_rel.deleted_at IS 'relation deleted at'; + + +CREATE TABLE IF NOT EXISTS group_meta ( + group_id BIGINT NOT NULL, + group_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (group_id), + UNIQUE (metalake_id, group_name, deleted_at) + ); +COMMENT ON TABLE group_meta IS 'group metadata'; + +COMMENT ON COLUMN group_meta.group_id IS 'group id'; +COMMENT ON COLUMN group_meta.group_name IS 'group name'; +COMMENT ON COLUMN group_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN group_meta.audit_info IS 'group audit info'; +COMMENT ON COLUMN group_meta.current_version IS 'group current version'; +COMMENT ON COLUMN group_meta.last_version IS 'group last version'; +COMMENT ON COLUMN group_meta.deleted_at IS 'group deleted at'; + + +CREATE TABLE IF NOT EXISTS group_role_rel ( + id BIGSERIAL NOT NULL, + group_id BIGINT NOT NULL, + role_id BIGINT NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (group_id, role_id, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_group_id ON group_role_rel (group_id); +COMMENT ON TABLE group_role_rel IS 'relation between group and role'; +COMMENT ON COLUMN group_role_rel.id IS 'auto increment id'; +COMMENT ON COLUMN group_role_rel.group_id IS 'group id'; +COMMENT ON COLUMN group_role_rel.role_id IS 'role id'; +COMMENT ON COLUMN group_role_rel.audit_info IS 'relation audit info'; +COMMENT ON COLUMN group_role_rel.current_version IS 'relation current version'; +COMMENT ON COLUMN group_role_rel.last_version IS 'relation last version'; +COMMENT ON COLUMN group_role_rel.deleted_at IS 'relation deleted at'; + +CREATE TABLE IF NOT EXISTS tag_meta ( + tag_id BIGINT NOT NULL, + tag_name VARCHAR(128) NOT NULL, + metalake_id BIGINT NOT NULL, + tag_comment VARCHAR(256) DEFAULT '', + properties TEXT DEFAULT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (tag_id), + UNIQUE (metalake_id, tag_name, deleted_at) + ); + +COMMENT ON TABLE tag_meta IS 'tag metadata'; + +COMMENT ON COLUMN tag_meta.tag_id IS 'tag id'; +COMMENT ON COLUMN tag_meta.tag_name IS 'tag name'; +COMMENT ON COLUMN tag_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN tag_meta.tag_comment IS 'tag comment'; +COMMENT ON COLUMN tag_meta.properties IS 'tag properties'; +COMMENT ON COLUMN tag_meta.audit_info IS 'tag audit info'; + + +CREATE TABLE IF NOT EXISTS tag_relation_meta ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY, + tag_id BIGINT NOT NULL, + metadata_object_id BIGINT NOT NULL, + metadata_object_type VARCHAR(64) NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (tag_id, metadata_object_id, metadata_object_type, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_tag_id ON tag_relation_meta (tag_id); +CREATE INDEX IF NOT EXISTS idx_metadata_object_id ON tag_relation_meta (metadata_object_id); +COMMENT ON TABLE tag_relation_meta IS 'tag metadata object relation'; +COMMENT ON COLUMN tag_relation_meta.id IS 'auto increment id'; +COMMENT ON COLUMN tag_relation_meta.tag_id IS 'tag id'; +COMMENT ON COLUMN tag_relation_meta.metadata_object_id IS 'metadata object id'; +COMMENT ON COLUMN tag_relation_meta.metadata_object_type IS 'metadata object type'; +COMMENT ON COLUMN tag_relation_meta.audit_info IS 'tag relation audit info'; +COMMENT ON COLUMN tag_relation_meta.current_version IS 'tag relation current version'; +COMMENT ON COLUMN tag_relation_meta.last_version IS 'tag relation last version'; +COMMENT ON COLUMN tag_relation_meta.deleted_at IS 'tag relation deleted at'; + +CREATE TABLE IF NOT EXISTS owner_meta ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY, + metalake_id BIGINT NOT NULL, + owner_id BIGINT NOT NULL, + owner_type VARCHAR(64) NOT NULL, + metadata_object_id BIGINT NOT NULL, + metadata_object_type VARCHAR(64) NOT NULL, + audit_info TEXT NOT NULL, + current_version INT NOT NULL DEFAULT 1, + last_version INT NOT NULL DEFAULT 1, + deleted_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (id), + UNIQUE (owner_id, metadata_object_id, metadata_object_type, deleted_at) + ); + +CREATE INDEX IF NOT EXISTS idx_owner_id ON owner_meta (owner_id); +CREATE INDEX IF NOT EXISTS idx_metadata_object_id ON owner_meta (metadata_object_id); +COMMENT ON TABLE owner_meta IS 'owner relation'; +COMMENT ON COLUMN owner_meta.id IS 'auto increment id'; +COMMENT ON COLUMN owner_meta.metalake_id IS 'metalake id'; +COMMENT ON COLUMN owner_meta.owner_id IS 'owner id'; +COMMENT ON COLUMN owner_meta.owner_type IS 'owner type'; +COMMENT ON COLUMN owner_meta.metadata_object_id IS 'metadata object id'; +COMMENT ON COLUMN owner_meta.metadata_object_type IS 'metadata object type'; +COMMENT ON COLUMN owner_meta.audit_info IS 'owner relation audit info'; +COMMENT ON COLUMN owner_meta.current_version IS 'owner relation current version'; +COMMENT ON COLUMN owner_meta.last_version IS 'owner relation last version'; +COMMENT ON COLUMN owner_meta.deleted_at IS 'owner relation deleted at'; + diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java index a8da0aa7e94..f7838d0ae83 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java @@ -252,7 +252,7 @@ public Response dropFileset( } @GET - @Path("{fileset}/fileLocation") + @Path("{fileset}/location") @Produces("application/vnd.gravitino.v1+json") @Timed(name = "get-file-location." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "get-file-location", absolute = true) @@ -261,7 +261,7 @@ public Response getFileLocation( @PathParam("catalog") String catalog, @PathParam("schema") String schema, @PathParam("fileset") String fileset, - @QueryParam("subPath") @NotNull String subPath) { + @QueryParam("sub_path") @NotNull String subPath) { LOG.info( "Received get file location request: {}.{}.{}.{}, sub path:{}", metalake, diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java index 8ee4ffd5964..c1badd3fb1f 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java @@ -439,8 +439,8 @@ public void testGetFileLocation() { when(dispatcher.getFileLocation(any(), any())).thenReturn(actualPath); Response resp = - target(filesetPath(metalake, catalog, schema) + "fileset1/fileLocation") - .queryParam("subPath", "test/1") + target(filesetPath(metalake, catalog, schema) + "fileset1/location") + .queryParam("sub_path", RESTUtils.encodeString("test/1")) .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") .get(); @@ -450,6 +450,35 @@ public void testGetFileLocation() { Assertions.assertEquals(0, contextResponse.getCode()); Assertions.assertEquals(actualPath, contextResponse.getFileLocation()); + + // Test throw NoSuchFilesetException + doThrow(new NoSuchFilesetException("no found")).when(dispatcher).getFileLocation(any(), any()); + Response resp1 = + target(filesetPath(metalake, catalog, schema) + "fileset1/location") + .queryParam("sub_path", RESTUtils.encodeString("test/1")) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchFilesetException.class.getSimpleName(), errorResp.getType()); + + // Test throw RuntimeException + doThrow(new RuntimeException("internal error")).when(dispatcher).getFileLocation(any(), any()); + Response resp2 = + target(filesetPath(metalake, catalog, schema) + "fileset1/location") + .queryParam("sub_path", RESTUtils.encodeString("test/1")) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp2 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); } private void assertUpdateFileset(FilesetUpdatesRequest req, Fileset updatedFileset) { diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java index c3b34bc6bff..6a4b8e95354 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestGroupOperations.java @@ -245,7 +245,7 @@ private Group buildGroup(String group) { return GroupEntity.builder() .withId(1L) .withName(group) - .withRoleNames(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build(); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java index 76dba3edd4b..39ed06ebbcf 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java @@ -52,6 +52,7 @@ import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; +import org.apache.gravitino.meta.RoleEntity; import org.apache.gravitino.meta.UserEntity; import org.apache.gravitino.rest.RESTUtils; import org.glassfish.hk2.utilities.binding.AbstractBinder; @@ -110,12 +111,18 @@ protected void configure() { @Test public void testGrantRolesToUser() { + RoleEntity roleEntity = + RoleEntity.builder() + .withId(1L) + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .withName("roles") + .build(); UserEntity userEntity = UserEntity.builder() .withId(1L) .withName("user") - .withRoleNames(Lists.newArrayList("roles")) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList(roleEntity)) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); @@ -207,12 +214,18 @@ public void testGrantRolesToUser() { @Test public void testGrantRolesToGroup() { + RoleEntity roleEntity = + RoleEntity.builder() + .withId(1L) + .withName("roles") + .withAuditInfo( + AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) + .build(); GroupEntity groupEntity = GroupEntity.builder() .withId(1L) .withName("group") - .withRoleNames(Lists.newArrayList("roles")) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList(roleEntity)) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); @@ -311,8 +324,7 @@ public void testRevokeRolesFromUser() { UserEntity.builder() .withId(1L) .withName("user") - .withRoleNames(Lists.newArrayList()) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); @@ -356,8 +368,7 @@ public void testRevokeRolesFromGroup() { GroupEntity.builder() .withId(1L) .withName("group") - .withRoleNames(Lists.newArrayList()) - .withRoleIds(Lists.newArrayList(1L)) + .withRoles(Lists.newArrayList()) .withAuditInfo( AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build()) .build(); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java index d3209e0e22c..b13c0175b0b 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestUserOperations.java @@ -246,7 +246,7 @@ private User buildUser(String user) { return UserEntity.builder() .withId(1L) .withName(user) - .withRoleNames(Collections.emptyList()) + .withRoles(Collections.emptyList()) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build();