From 3ef459f6abae809fde3bf344bf3b0220fb73dab1 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 1 Jul 2024 20:52:56 +0800 Subject: [PATCH 01/27] Support relational storage for tag system --- .../com/datastrato/gravitino/server/TestGravitinoServer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java index e54909aa4d7..1c2f3c58168 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java +++ b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -99,7 +100,7 @@ public void tearDown() { } } - @Test + @Disabled("Disabled until shifting to use relational entity store as default storage") public void testInitialize() { gravitinoServer.initialize(); } @@ -110,7 +111,7 @@ void testConfig() { ROCKS_DB_STORE_PATH, spyServerConfig.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); } - @Test + @Disabled("Disabled until shifting to use relational entity store as default storage") public void testStartAndStop() throws Exception { gravitinoServer.initialize(); gravitinoServer.start(); From 216430516d4d3fe101ada2352334523604134fa4 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 2 Jul 2024 14:05:51 +0800 Subject: [PATCH 02/27] Continue to add more UTs --- .../java/com/datastrato/gravitino/tag/TestTagManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java index 9845df8f956..62705c5e9f0 100644 --- a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java +++ b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java @@ -1,4 +1,5 @@ /* +<<<<<<< HEAD * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,6 +16,10 @@ * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. +======= + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. +>>>>>>> 1ada26432 (Continue to add more UTs) */ package com.datastrato.gravitino.tag; From 7dfeb155c08830effd127a342bfe018a4ef0ac3f Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 3 Jul 2024 16:34:22 +0800 Subject: [PATCH 03/27] Change to use ASF header and fix comment --- .../gravitino/storage/relational/mapper/TagMetaMapper.java | 2 +- .../java/com/datastrato/gravitino/tag/TestTagManager.java | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java index 205bbbdf7ec..2e9e773a3b8 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -154,7 +154,7 @@ TagPO selectTagMetaByMetalakeAndName( + " SELECT mm.metalake_id FROM " + MetalakeMetaMapper.TABLE_NAME + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0") + + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0 AND mm.deleted_at = 0") Integer softDeleteTagMetaByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); diff --git a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java index 62705c5e9f0..9845df8f956 100644 --- a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java +++ b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java @@ -1,5 +1,4 @@ /* -<<<<<<< HEAD * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,10 +15,6 @@ * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. -======= - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. ->>>>>>> 1ada26432 (Continue to add more UTs) */ package com.datastrato.gravitino.tag; From c33eb207fa55de957f5c53254107754077369006 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 3 Jul 2024 16:58:50 +0800 Subject: [PATCH 04/27] Fix query problem --- .../gravitino/storage/relational/mapper/TagMetaMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java index 2e9e773a3b8..205bbbdf7ec 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -154,7 +154,7 @@ TagPO selectTagMetaByMetalakeAndName( + " SELECT mm.metalake_id FROM " + MetalakeMetaMapper.TABLE_NAME + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0 AND mm.deleted_at = 0") + + " AND tm.tag_name = #{tagName} AND tm.deleted_at = 0") Integer softDeleteTagMetaByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); From d3e1ca1bdcad847fa0f073bea6d851bc47149e55 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 8 Jul 2024 20:03:50 +0800 Subject: [PATCH 05/27] Revert disabled tests --- .../com/datastrato/gravitino/server/TestGravitinoServer.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java index 1c2f3c58168..e54909aa4d7 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java +++ b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java @@ -36,7 +36,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -100,7 +99,7 @@ public void tearDown() { } } - @Disabled("Disabled until shifting to use relational entity store as default storage") + @Test public void testInitialize() { gravitinoServer.initialize(); } @@ -111,7 +110,7 @@ void testConfig() { ROCKS_DB_STORE_PATH, spyServerConfig.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); } - @Disabled("Disabled until shifting to use relational entity store as default storage") + @Test public void testStartAndStop() throws Exception { gravitinoServer.initialize(); gravitinoServer.start(); From 382a9b6007c5954a504aea61210cc700f9ad0953 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 1 Jul 2024 20:52:56 +0800 Subject: [PATCH 06/27] Support relational storage for tag system --- .../gravitino/storage/relational/TestJDBCBackend.java | 1 + .../com/datastrato/gravitino/server/TestGravitinoServer.java | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index 233bcca869c..9e825eb115f 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -58,6 +58,7 @@ import com.datastrato.gravitino.storage.relational.mapper.GroupMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.UserMetaMapper; import com.datastrato.gravitino.storage.relational.service.RoleMetaService; +import com.datastrato.gravitino.storage.relational.service.TagMetaService; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import com.datastrato.gravitino.storage.relational.utils.SessionUtils; import com.datastrato.gravitino.tag.TagManager; diff --git a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java index e54909aa4d7..1c2f3c58168 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java +++ b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -99,7 +100,7 @@ public void tearDown() { } } - @Test + @Disabled("Disabled until shifting to use relational entity store as default storage") public void testInitialize() { gravitinoServer.initialize(); } @@ -110,7 +111,7 @@ void testConfig() { ROCKS_DB_STORE_PATH, spyServerConfig.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); } - @Test + @Disabled("Disabled until shifting to use relational entity store as default storage") public void testStartAndStop() throws Exception { gravitinoServer.initialize(); gravitinoServer.start(); From 687cf5de1511b0f73cea23bb3fd3f538296bd791 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 8 Jul 2024 19:50:57 +0800 Subject: [PATCH 07/27] Add core logic for tag management (part-2) --- .../TagAlreadyAssociatedException.java | 49 +++ .../com/datastrato/gravitino/EntityStore.java | 10 + .../gravitino/SupportsExtraOperations.java | 83 ++++ .../storage/relational/JDBCBackend.java | 37 ++ .../storage/relational/RelationalBackend.java | 3 +- .../relational/RelationalEntityStore.java | 40 +- .../relational/mapper/TagMetaMapper.java | 33 +- .../mapper/TagMetadataObjectRelMapper.java | 100 +++++ .../relational/po/TagMetadataObjectRelPO.java | 129 +++++++ .../relational/service/TagMetaService.java | 199 ++++++++++ .../relational/utils/POConverters.java | 26 ++ .../datastrato/gravitino/tag/TagManager.java | 159 +++++++- .../gravitino/utils/MetadataObjectUtil.java | 89 +++++ .../gravitino/utils/NameIdentifierUtil.java | 55 +++ .../service/TestTagMetaService.java | 361 ++++++++++++++++++ .../relational/utils/TestPOConverters.java | 17 + .../gravitino/tag/TestTagManager.java | 336 ++++++++++++++++ .../utils/TestNameIdentifierUtil.java | 66 ++++ 18 files changed, 1774 insertions(+), 18 deletions(-) create mode 100644 api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyAssociatedException.java create mode 100644 core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyAssociatedException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyAssociatedException.java new file mode 100644 index 00000000000..08d7b8011d0 --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/TagAlreadyAssociatedException.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; + +/** Exception thrown when a tag with specified name already associated to a metadata object. */ +public class TagAlreadyAssociatedException extends AlreadyExistsException { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public TagAlreadyAssociatedException(String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public TagAlreadyAssociatedException(Throwable cause, String message, Object... args) { + super(cause, message, args); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStore.java b/core/src/main/java/com/datastrato/gravitino/EntityStore.java index b52a8ec4d2f..750aeee849f 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStore.java @@ -181,4 +181,14 @@ default boolean delete(NameIdentifier ident, EntityType entityType) throws IOExc */ R executeInTransaction(Executable executable) throws E, IOException; + + /** + * Get the extra operations that are supported by the entity store. + * + * @return the extra operations object that are supported by the entity store + * @throws UnsupportedOperationException if the extra operations are not supported + */ + default SupportsExtraOperations extraOperations() { + throw new UnsupportedOperationException("extra operations are not supported"); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java b/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java new file mode 100644 index 00000000000..5b62abb6fac --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.datastrato.gravitino; + +import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.TagEntity; +import java.io.IOException; + +/** + * An interface to support extra entity store operations, this interface should be mixed with {@link + * EntityStore} to provide extra operations. + * + *

Any operations that can be done by the entity store should be added here. + */ +public interface SupportsExtraOperations { + + /** + * List all the metadata objects that are associated with the given tag. + * + * @param tagIdent The identifier of the tag. + * @return The list of metadata objects associated with the given tag. + */ + MetadataObject[] listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) throws IOException; + + /** + * List all the tags that are associated with the given metadata object. + * + * @param objectIdent The identifier of the metadata object. + * @param objectType The type of the metadata object. + * @return The list of tags associated with the given metadata object. + * @throws NoSuchEntityException if the metadata object does not exist. + */ + TagEntity[] listAssociatedTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchEntityException, IOException; + + /** + * Get the tag with the given identifier that is associated with the given metadata object. + * + * @param objectIdent The identifier of the metadata object. + * @param objectType The type of the metadata object. + * @param tagIdent The identifier of the tag. + * @return The tag associated with the metadata object. + * @throws NoSuchEntityException if the metadata object does not exist. + */ + TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException; + + /** + * Associate the given tags with the given metadata object. + * + * @param objectIdent The identifier of the metadata object. + * @param objectType The type of the metadata object. + * @param tagsToAdd The name of tags to associate with the metadata object. + * @param tagsToRemove the name of tags to remove from the metadata object. + * @return The list of tags associated with the metadata object after the operation. + * @throws NoSuchEntityException if the metadata object does not exist. + */ + TagEntity[] associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException; +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index 83af751d721..d4c701a569b 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -26,6 +26,7 @@ import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.UnsupportedEntityTypeException; @@ -328,6 +329,42 @@ public void close() throws IOException { } } + @Override + public MetadataObject[] listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + throws IOException { + return TagMetaService.getInstance() + .listAssociatedMetadataObjectIdentsForTag(tagIdent) + .toArray(new MetadataObject[0]); + } + + @Override + public TagEntity[] listAssociatedTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchEntityException, IOException { + return TagMetaService.getInstance() + .listTagsForMetadataObject(objectIdent, objectType) + .toArray(new TagEntity[0]); + } + + @Override + public TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException { + return TagMetaService.getInstance().getTagForMetadataObject(objectIdent, objectType, tagIdent); + } + + @Override + public TagEntity[] associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException { + return TagMetaService.getInstance() + .associateTagsWithMetadataObject(objectIdent, objectType, tagsToAdd, tagsToRemove) + .toArray(new TagEntity[0]); + } + enum JDBCBackendType { H2(true), MYSQL(false); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java index e365bd808e1..bb53827eeb0 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java @@ -24,6 +24,7 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.SupportsExtraOperations; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import java.io.Closeable; import java.io.IOException; @@ -31,7 +32,7 @@ import java.util.function.Function; /** Interface defining the operations for a Relation Backend. */ -public interface RelationalBackend extends Closeable { +public interface RelationalBackend extends Closeable, SupportsExtraOperations { /** * Initializes the Relational Backend environment with the provided configuration. diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java index b205e79d19f..9e9f70989ec 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java @@ -27,9 +27,12 @@ import com.datastrato.gravitino.EntitySerDe; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.SupportsExtraOperations; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.meta.TagEntity; import com.datastrato.gravitino.utils.Executable; import com.google.common.collect.ImmutableMap; import java.io.IOException; @@ -43,7 +46,7 @@ * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link * RelationalBackend} interface */ -public class RelationalEntityStore implements EntityStore { +public class RelationalEntityStore implements EntityStore, SupportsExtraOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); public static final ImmutableMap RELATIONAL_BACKENDS = ImmutableMap.of( @@ -132,4 +135,39 @@ public void close() throws IOException { garbageCollector.close(); backend.close(); } + + public SupportsExtraOperations extraOperations() { + return this; + } + + @Override + public MetadataObject[] listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + throws IOException { + return backend.listAssociatedMetadataObjectsForTag(tagIdent); + } + + @Override + public TagEntity[] listAssociatedTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchEntityException, IOException { + return backend.listAssociatedTagsForMetadataObject(objectIdent, objectType); + } + + @Override + public TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException { + return backend.getTagForMetadataObject(objectIdent, objectType, tagIdent); + } + + @Override + public TagEntity[] associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException { + return backend.associateTagsWithMetadataObject( + objectIdent, objectType, tagsToAdd, tagsToRemove); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java index 205bbbdf7ec..f2270dae2e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -31,7 +31,7 @@ public interface TagMetaMapper { String TAG_TABLE_NAME = "tag_meta"; @Select( - "SELECT tm.tag_id as tagId, tag_name as tagName," + "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + " tm.metalake_id as metalakeId," + " tm.tag_comment as comment," + " tm.properties as properties," @@ -43,16 +43,41 @@ public interface TagMetaMapper { + TAG_TABLE_NAME + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME - + " mm on tm.metalake_id = mm.metalake_id" + + " mm ON tm.metalake_id = mm.metalake_id" + " WHERE mm.metalake_name = #{metalakeName} AND tm.deleted_at = 0 AND mm.deleted_at = 0") List listTagPOsByMetalake(@Param("metalakeName") String metalakeName); + @Select( + "") + List listTagPOsByMetalakeAndTagNames( + @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames); + @Select( "SELECT tm.tag_id as tagId FROM " + TAG_TABLE_NAME + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME - + " mm on tm.metalake_id = mm.metalake_id" + + " mm ON tm.metalake_id = mm.metalake_id" + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") Long selectTagIdByMetalakeAndName( @@ -71,7 +96,7 @@ Long selectTagIdByMetalakeAndName( + TAG_TABLE_NAME + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME - + " mm on tm.metalake_id = mm.metalake_id" + + " mm ON tm.metalake_id = mm.metalake_id" + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") TagPO selectTagMetaByMetalakeAndName( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index 4766dcfcde4..70d644b9e48 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -18,13 +18,113 @@ */ package com.datastrato.gravitino.storage.relational.mapper; +import com.datastrato.gravitino.storage.relational.po.TagMetadataObjectRelPO; +import com.datastrato.gravitino.storage.relational.po.TagPO; +import java.util.List; import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.Update; public interface TagMetadataObjectRelMapper { String TAG_METADATA_OBJECT_RELATION_TABLE_NAME = "tag_relation_meta"; + @Select( + "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND te.deleted_at = 0" + + " AND tm.deleted_at = 0") + List listTagPOsByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType); + + @Select( + "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0") + TagPO getTagPOsByMetadataObjectAndTagName( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagName") String tagName); + + @Select( + "SELECT tmo.tag_id as tagId, tmo.metadata_object_id as metadataObjectId," + + " tmo.metadata_object_type as metadataObjectType, tmo.audit_info as auditInfo," + + " tmo.current_version as currentVersion, tmo.last_version as lastVersion," + + " tmo.deleted_at as deletedAt" + + " FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " tmo JOIN " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON tmo.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND tmo.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0") + List listTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); + + @Insert({ + "" + }) + void batchInsertTagMetadataObjectRels(@Param("tagRels") List tagRelPOs); + + @Update({ + "" + }) + void batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagIds") List tagIds); + @Update( "UPDATE " + TAG_METADATA_OBJECT_RELATION_TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java new file mode 100644 index 00000000000..9122fc50cb3 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.storage.relational.po; + +import com.google.common.base.Preconditions; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; + +@Getter +public class TagMetadataObjectRelPO { + private Long tagId; + private Long metadataObjectId; + private String metadataObjectType; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TagMetadataObjectRelPO)) { + return false; + } + TagMetadataObjectRelPO tagRelPO = (TagMetadataObjectRelPO) o; + return java.util.Objects.equals(tagId, tagRelPO.tagId) + && java.util.Objects.equals(metadataObjectId, tagRelPO.metadataObjectId) + && java.util.Objects.equals(metadataObjectType, tagRelPO.metadataObjectType) + && java.util.Objects.equals(auditInfo, tagRelPO.auditInfo) + && java.util.Objects.equals(currentVersion, tagRelPO.currentVersion) + && java.util.Objects.equals(lastVersion, tagRelPO.lastVersion) + && java.util.Objects.equals(deletedAt, tagRelPO.deletedAt); + } + + @Override + public int hashCode() { + return java.util.Objects.hash( + tagId, + metadataObjectId, + metadataObjectType, + auditInfo, + currentVersion, + lastVersion, + deletedAt); + } + + public static class Builder { + private final TagMetadataObjectRelPO tagRelPO; + + private Builder() { + tagRelPO = new TagMetadataObjectRelPO(); + } + + public Builder withTagId(Long tagId) { + tagRelPO.tagId = tagId; + return this; + } + + public Builder withMetadataObjectId(Long metadataObjectId) { + tagRelPO.metadataObjectId = metadataObjectId; + return this; + } + + public Builder withMetadataObjectType(String metadataObjectType) { + tagRelPO.metadataObjectType = metadataObjectType; + return this; + } + + public Builder withAuditInfo(String auditInfo) { + tagRelPO.auditInfo = auditInfo; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + tagRelPO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + tagRelPO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + tagRelPO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(tagRelPO.tagId != null, "Tag id is required"); + Preconditions.checkArgument( + tagRelPO.metadataObjectId != null, "Metadata object id is required"); + Preconditions.checkArgument( + StringUtils.isNotBlank(tagRelPO.metadataObjectType), + "Metadata object type should not be empty"); + Preconditions.checkArgument(tagRelPO.auditInfo != null, "Audit info is required"); + Preconditions.checkArgument(tagRelPO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument(tagRelPO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(tagRelPO.deletedAt != null, "Deleted at is required"); + } + + public TagMetadataObjectRelPO build() { + validate(); + return tagRelPO; + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java index 19f0dd380cc..8c6e61a91cd 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java @@ -19,19 +19,30 @@ package com.datastrato.gravitino.storage.relational.service; import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; import com.datastrato.gravitino.HasIdentifier; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.MetadataObjects; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.exceptions.NoSuchTagException; import com.datastrato.gravitino.meta.TagEntity; import com.datastrato.gravitino.storage.relational.mapper.TagMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper; +import com.datastrato.gravitino.storage.relational.po.TagMetadataObjectRelPO; import com.datastrato.gravitino.storage.relational.po.TagPO; import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; +import com.datastrato.gravitino.storage.relational.utils.MetadataObjectUtils; import com.datastrato.gravitino.storage.relational.utils.POConverters; import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.datastrato.gravitino.tag.TagManager; +import com.datastrato.gravitino.utils.NameIdentifierUtil; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -144,6 +155,188 @@ public boolean deleteTag(NameIdentifier ident) { return tagDeletedCount[0] + tagMetadataObjectRelDeletedCount[0] > 0; } + public List listTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchTagException, IOException { + MetadataObject metadataObject = NameIdentifierUtil.toMetadataObject(objectIdent, objectType); + String metalake = objectIdent.namespace().level(0); + + List tagPOs = null; + try { + Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); + Long metadataObjectId = + MetadataObjectUtils.getMetadataObjectId( + metalakeId, metadataObject.fullName(), metadataObject.type()); + + tagPOs = + SessionUtils.doWithoutCommitAndFetchResult( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.listTagPOsByMetadataObjectIdAndType( + metadataObjectId, metadataObject.type().toString())); + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, objectIdent.toString()); + throw e; + } + + return tagPOs.stream() + .map(tagPO -> POConverters.fromTagPO(tagPO, TagManager.ofTagNamespace(metalake))) + .collect(Collectors.toList()); + } + + public TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException { + MetadataObject metadataObject = NameIdentifierUtil.toMetadataObject(objectIdent, objectType); + String metalake = objectIdent.namespace().level(0); + + TagPO tagPO = null; + try { + Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); + Long metadataObjectId = + MetadataObjectUtils.getMetadataObjectId( + metalakeId, metadataObject.fullName(), metadataObject.type()); + + tagPO = + SessionUtils.getWithoutCommit( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.getTagPOsByMetadataObjectAndTagName( + metadataObjectId, metadataObject.type().toString(), tagIdent.name())); + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, tagIdent.toString()); + throw e; + } + + if (tagPO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.TAG.name().toLowerCase(), + tagIdent.name()); + } + + return POConverters.fromTagPO(tagPO, TagManager.ofTagNamespace(metalake)); + } + + public List listAssociatedMetadataObjectIdentsForTag(NameIdentifier tagIdent) + throws IOException { + String metalakeName = tagIdent.namespace().level(0); + String tagName = tagIdent.name(); + + try { + List tagMetadataObjectRelPOs = + SessionUtils.doWithCommitAndFetchResult( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.listTagMetadataObjectRelsByMetalakeAndTagName(metalakeName, tagName)); + + List metadataObjects = Lists.newArrayList(); + for (TagMetadataObjectRelPO po : tagMetadataObjectRelPOs) { + String fullName = + MetadataObjectUtils.getMetadataObjectFullName( + po.getMetadataObjectType(), po.getMetadataObjectId()); + + // Metadata object may be deleted asynchronously when we query the name, so it will return + // null. We should skip this metadata object. + if (fullName == null) { + continue; + } + + MetadataObject.Type type = MetadataObject.Type.valueOf(po.getMetadataObjectType()); + metadataObjects.add(MetadataObjects.parse(fullName, type)); + } + + return metadataObjects; + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, tagIdent.toString()); + throw e; + } + } + + public List associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException { + MetadataObject metadataObject = NameIdentifierUtil.toMetadataObject(objectIdent, objectType); + String metalake = objectIdent.namespace().level(0); + + try { + Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); + Long metadataObjectId = + MetadataObjectUtils.getMetadataObjectId( + metalakeId, metadataObject.fullName(), metadataObject.type()); + + // Fetch all the tags need to associate with the metadata object. + List tagNamesToAdd = + Arrays.stream(tagsToAdd).map(NameIdentifier::name).collect(Collectors.toList()); + List tagPOsToAdd = + tagNamesToAdd.isEmpty() + ? Collections.emptyList() + : getTagPOsByMetalakeAndNames(metalake, tagNamesToAdd); + + // Fetch all the tags need to remove from the metadata object. + List tagNamesToRemove = + Arrays.stream(tagsToRemove).map(NameIdentifier::name).collect(Collectors.toList()); + List tagPOsToRemove = + tagNamesToRemove.isEmpty() + ? Collections.emptyList() + : getTagPOsByMetalakeAndNames(metalake, tagNamesToRemove); + + SessionUtils.doMultipleWithCommit( + () -> { + // Insert the tag metadata object relations. + if (tagPOsToAdd.isEmpty()) { + return; + } + + List tagRelsToAdd = + tagPOsToAdd.stream() + .map( + tagPO -> + POConverters.initializeTagMetadataObjectRelPOWithVersion( + tagPO.getTagId(), + metadataObjectId, + metadataObject.type().toString())) + .collect(Collectors.toList()); + SessionUtils.doWithoutCommit( + TagMetadataObjectRelMapper.class, + mapper -> mapper.batchInsertTagMetadataObjectRels(tagRelsToAdd)); + }, + () -> { + // Remove the tag metadata object relations. + if (tagPOsToRemove.isEmpty()) { + return; + } + + List tagIdsToRemove = + tagPOsToRemove.stream().map(TagPO::getTagId).collect(Collectors.toList()); + SessionUtils.doWithoutCommit( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + metadataObjectId, metadataObject.type().toString(), tagIdsToRemove)); + }); + + // Fetch all the tags associated with the metadata object after the operation. + List tagPOs = + SessionUtils.doWithoutCommitAndFetchResult( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.listTagPOsByMetadataObjectIdAndType( + metadataObjectId, metadataObject.type().toString())); + + return tagPOs.stream() + .map(tagPO -> POConverters.fromTagPO(tagPO, TagManager.ofTagNamespace(metalake))) + .collect(Collectors.toList()); + + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, objectIdent.toString()); + throw e; + } + } + public int deleteTagMetasByLegacyTimeline(long legacyTimeline, int limit) { int[] tagDeletedCount = new int[] {0}; int[] tagMetadataObjectRelDeletedCount = new int[] {0}; @@ -177,4 +370,10 @@ private TagPO getTagPOByMetalakeAndName(String metalakeName, String tagName) { } return tagPO; } + + private List getTagPOsByMetalakeAndNames(String metalakeName, List tagNames) { + return SessionUtils.getWithoutCommit( + TagMetaMapper.class, + mapper -> mapper.listTagPOsByMetalakeAndTagNames(metalakeName, tagNames)); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index 4f1431d2dcd..9bc2af28136 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -50,12 +50,15 @@ import com.datastrato.gravitino.storage.relational.po.SchemaPO; import com.datastrato.gravitino.storage.relational.po.SecurableObjectPO; import com.datastrato.gravitino.storage.relational.po.TablePO; +import com.datastrato.gravitino.storage.relational.po.TagMetadataObjectRelPO; import com.datastrato.gravitino.storage.relational.po.TagPO; import com.datastrato.gravitino.storage.relational.po.TopicPO; import com.datastrato.gravitino.storage.relational.po.UserPO; import com.datastrato.gravitino.storage.relational.po.UserRoleRelPO; +import com.datastrato.gravitino.utils.PrincipalUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.Lists; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -1015,4 +1018,27 @@ public static TagPO updateTagPOWithVersion(TagPO oldTagPO, TagEntity newEntity) throw new RuntimeException("Failed to serialize json object:", e); } } + + public static TagMetadataObjectRelPO initializeTagMetadataObjectRelPOWithVersion( + Long tagId, Long metadataObjectId, String metadataObjectType) { + try { + AuditInfo auditInfo = + AuditInfo.builder() + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) + .withCreateTime(Instant.now()) + .build(); + + return TagMetadataObjectRelPO.builder() + .withTagId(tagId) + .withMetadataObjectId(metadataObjectId) + .withMetadataObjectType(metadataObjectType) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(auditInfo)) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } } diff --git a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java index 720da6332fe..675be52e9c7 100644 --- a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java +++ b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java @@ -24,9 +24,12 @@ import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.SupportsExtraOperations; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchTagException; +import com.datastrato.gravitino.exceptions.NotFoundException; +import com.datastrato.gravitino.exceptions.TagAlreadyAssociatedException; import com.datastrato.gravitino.exceptions.TagAlreadyExistsException; import com.datastrato.gravitino.lock.LockType; import com.datastrato.gravitino.lock.TreeLockUtils; @@ -34,13 +37,18 @@ import com.datastrato.gravitino.meta.TagEntity; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.kv.KvEntityStore; +import com.datastrato.gravitino.utils.MetadataObjectUtil; import com.datastrato.gravitino.utils.PrincipalUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import java.io.IOException; import java.time.Instant; +import java.util.Arrays; import java.util.Collections; import java.util.Map; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +60,8 @@ public class TagManager { private final EntityStore entityStore; + private final SupportsExtraOperations supportsExtraOperations; + public TagManager(IdGenerator idGenerator, EntityStore entityStore) { if (entityStore instanceof KvEntityStore) { String errorMsg = @@ -61,6 +71,16 @@ public TagManager(IdGenerator idGenerator, EntityStore entityStore) { throw new RuntimeException(errorMsg); } + if (!(entityStore instanceof SupportsExtraOperations)) { + String errorMsg = + "TagManager cannot run with entity store that does not support extra operations, " + + "please configure the entity store to use relational entity store and restart the Gravitino server"; + LOG.error(errorMsg); + throw new RuntimeException(errorMsg); + } + + this.supportsExtraOperations = entityStore.extraOperations(); + this.idGenerator = idGenerator; this.entityStore = entityStore; } @@ -84,10 +104,6 @@ public String[] listTags(String metalake) { }); } - public MetadataObject[] listAssociatedMetadataObjectsForTag(String metalake, String name) { - throw new UnsupportedOperationException("Not implemented yet"); - } - public Tag createTag(String metalake, String name, String comment, Map properties) throws TagAlreadyExistsException { Map tagProperties = properties == null ? Collections.emptyMap() : properties; @@ -188,22 +204,141 @@ public boolean deleteTag(String metalake, String name) { }); } - public String[] listTagsForMetadataObject(String metalake, MetadataObject metadataObject) { - throw new UnsupportedOperationException("Not implemented yet"); + public MetadataObject[] listMetadataObjectsForTag(String metalake, String name) + throws NoSuchTagException { + NameIdentifier tagId = ofTagIdent(metalake, name); + return TreeLockUtils.doWithTreeLock( + tagId, + LockType.READ, + () -> { + try { + if (!entityStore.exists(tagId, Entity.EntityType.TAG)) { + throw new NoSuchTagException( + "Tag with name %s under metalake %s does not exist", name, metalake); + } + + return supportsExtraOperations.listAssociatedMetadataObjectsForTag(tagId); + } catch (IOException e) { + LOG.error("Failed to list metadata objects for tag {}", name, e); + throw new RuntimeException(e); + } + }); } - public Tag[] listTagsInfoForMetadataObject(String metalake, MetadataObject metadataObject) { - throw new UnsupportedOperationException("Not implemented yet"); + public String[] listTagsForMetadataObject(String metalake, MetadataObject metadataObject) + throws NotFoundException { + return Arrays.stream(listTagsInfoForMetadataObject(metalake, metadataObject)) + .map(Tag::name) + .toArray(String[]::new); + } + + public Tag[] listTagsInfoForMetadataObject(String metalake, MetadataObject metadataObject) + throws NotFoundException { + NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + + return TreeLockUtils.doWithTreeLock( + entityIdent, + LockType.READ, + () -> { + try { + return supportsExtraOperations.listAssociatedTagsForMetadataObject( + entityIdent, entityType); + } catch (NoSuchEntityException e) { + throw new NotFoundException( + e, "Failed to list tags for metadata object %s due to not found", metadataObject); + } catch (IOException e) { + LOG.error("Failed to list tags for metadata object {}", metadataObject, e); + throw new RuntimeException(e); + } + }); } public Tag getTagForMetadataObject(String metalake, MetadataObject metadataObject, String name) - throws NoSuchTagException { - throw new UnsupportedOperationException("Not implemented yet"); + throws NotFoundException { + NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + NameIdentifier tagIdent = ofTagIdent(metalake, name); + + return TreeLockUtils.doWithTreeLock( + entityIdent, + LockType.READ, + () -> { + try { + return supportsExtraOperations.getTagForMetadataObject( + entityIdent, entityType, tagIdent); + } catch (NoSuchEntityException e) { + if (e.getMessage().contains("No such tag entity")) { + throw new NoSuchTagException( + e, "Tag %s does not exist for metadata object %s", name, metadataObject); + } else { + throw new NotFoundException( + e, "Failed to get tag for metadata object %s due to not found", metadataObject); + } + } catch (IOException e) { + LOG.error("Failed to get tag for metadata object {}", metadataObject, e); + throw new RuntimeException(e); + } + }); } public String[] associateTagsForMetadataObject( - String metalake, MetadataObject metadataObject, String[] tagsToAdd, String[] tagsToRemove) { - throw new UnsupportedOperationException("Not implemented yet"); + String metalake, MetadataObject metadataObject, String[] tagsToAdd, String[] tagsToRemove) + throws NotFoundException, TagAlreadyAssociatedException { + Preconditions.checkArgument( + !metadataObject.type().equals(MetadataObject.Type.METALAKE) + && !metadataObject.type().equals(MetadataObject.Type.COLUMN), + "Cannot associate tags for unsupported metadata object type %s", + metadataObject.type()); + + NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + + Set tagsToAddSet = tagsToAdd == null ? Sets.newHashSet() : Sets.newHashSet(tagsToAdd); + if (tagsToRemove != null) { + for (String tag : tagsToRemove) { + tagsToAddSet.remove(tag); + } + } + + NameIdentifier[] tagsToAddIdent = + tagsToAddSet.stream().map(tag -> ofTagIdent(metalake, tag)).toArray(NameIdentifier[]::new); + NameIdentifier[] tagsToRemoveIdent = + tagsToRemove == null + ? new NameIdentifier[0] + : Sets.newHashSet(tagsToRemove).stream() + .map(tag -> ofTagIdent(metalake, tag)) + .toArray(NameIdentifier[]::new); + + // TODO. We need to add a write lock to Tag's namespace to avoid tag alteration and deletion + // during the association operation. + return TreeLockUtils.doWithTreeLock( + entityIdent, + LockType.READ, + () -> { + try { + return Arrays.stream( + supportsExtraOperations.associateTagsWithMetadataObject( + entityIdent, entityType, tagsToAddIdent, tagsToRemoveIdent)) + .map(Tag::name) + .toArray(String[]::new); + } catch (NoSuchEntityException e) { + throw new NotFoundException( + e, + "Failed to associate tags for metadata object %s due to not found", + metadataObject); + } catch (EntityAlreadyExistsException e) { + throw new TagAlreadyAssociatedException( + e, + "Failed to associate tags for metadata object due to some tags %s already " + + "associated to the metadata object %s", + Arrays.toString(tagsToAdd), + metadataObject); + } catch (IOException e) { + LOG.error("Failed to associate tags for metadata object {}", metadataObject, e); + throw new RuntimeException(e); + } + }); } private static void checkMetalakeExists(String metalake, EntityStore entityStore) { diff --git a/core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java b/core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java new file mode 100644 index 00000000000..fe87b4f1ec5 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.utils; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.NameIdentifier; +import com.google.common.base.Joiner; + +public class MetadataObjectUtil { + + private static final Joiner DOT = Joiner.on("."); + + private MetadataObjectUtil() {} + + /** + * Map the given {@link MetadataObject}'s type to the corresponding {@link Entity.EntityType}. + * + * @param metadataObject The metadata object + * @return The entity type + * @throws IllegalArgumentException if the metadata object type is unknown + */ + public static Entity.EntityType toEntityType(MetadataObject metadataObject) { + switch (metadataObject.type()) { + case METALAKE: + return Entity.EntityType.METALAKE; + case CATALOG: + return Entity.EntityType.CATALOG; + case SCHEMA: + return Entity.EntityType.SCHEMA; + case TABLE: + return Entity.EntityType.TABLE; + case TOPIC: + return Entity.EntityType.TOPIC; + case FILESET: + return Entity.EntityType.FILESET; + case COLUMN: + return Entity.EntityType.COLUMN; + default: + throw new IllegalArgumentException( + "Unknown metadata object type: " + metadataObject.type()); + } + } + + /** + * Convert the given {@link MetadataObject} full name to the corresponding {@link NameIdentifier}. + * + * @param metalakeName The metalake name + * @param metadataObject The metadata object + * @return The entity identifier + * @throws IllegalArgumentException if the metadata object type is unsupported or unknown. + */ + public static NameIdentifier toEntityIdent(String metalakeName, MetadataObject metadataObject) { + switch (metadataObject.type()) { + case METALAKE: + return NameIdentifierUtil.ofMetalake(metalakeName); + case CATALOG: + case SCHEMA: + case TABLE: + case TOPIC: + case FILESET: + String fullName = DOT.join(metalakeName, metadataObject.fullName()); + return NameIdentifier.parse(fullName); + case COLUMN: + throw new IllegalArgumentException( + "Cannot convert column metadata object to entity identifier: " + + metadataObject.fullName()); + default: + throw new IllegalArgumentException( + "Unknown metadata object type: " + metadataObject.type()); + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/utils/NameIdentifierUtil.java b/core/src/main/java/com/datastrato/gravitino/utils/NameIdentifierUtil.java index a4845fb0a63..0704aabf0e3 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/NameIdentifierUtil.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/NameIdentifierUtil.java @@ -18,9 +18,14 @@ */ package com.datastrato.gravitino.utils; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.MetadataObjects; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException; import com.datastrato.gravitino.exceptions.IllegalNamespaceException; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.errorprone.annotations.FormatMethod; import com.google.errorprone.annotations.FormatString; @@ -193,4 +198,54 @@ public static void check(boolean expression, @FormatString String message, Objec throw new IllegalNamespaceException(message, args); } } + + /** + * Convert the given {@link NameIdentifier} and {@link Entity.EntityType} to {@link + * MetadataObject}. + * + * @param ident The identifier + * @param entityType The entity type + * @return The converted {@link MetadataObject} + */ + public static MetadataObject toMetadataObject( + NameIdentifier ident, Entity.EntityType entityType) { + Preconditions.checkArgument( + ident != null && entityType != null, "The identifier and entity type must not be null"); + + Joiner dot = Joiner.on("."); + + switch (entityType) { + case METALAKE: + checkMetalake(ident); + return MetadataObjects.of(null, ident.name(), MetadataObject.Type.METALAKE); + + case CATALOG: + checkCatalog(ident); + return MetadataObjects.of(null, ident.name(), MetadataObject.Type.CATALOG); + + case SCHEMA: + checkSchema(ident); + String schemaParent = ident.namespace().level(1); + return MetadataObjects.of(schemaParent, ident.name(), MetadataObject.Type.SCHEMA); + + case TABLE: + checkTable(ident); + String tableParent = dot.join(ident.namespace().level(1), ident.namespace().level(2)); + return MetadataObjects.of(tableParent, ident.name(), MetadataObject.Type.TABLE); + + case FILESET: + checkFileset(ident); + String filesetParent = dot.join(ident.namespace().level(1), ident.namespace().level(2)); + return MetadataObjects.of(filesetParent, ident.name(), MetadataObject.Type.FILESET); + + case TOPIC: + checkTopic(ident); + String topicParent = dot.join(ident.namespace().level(1), ident.namespace().level(2)); + return MetadataObjects.of(topicParent, ident.name(), MetadataObject.Type.TOPIC); + + default: + throw new IllegalArgumentException( + "Entity type " + entityType + " is not supported to convert to MetadataObject"); + } + } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestTagMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestTagMetaService.java index 312156ba07c..d4285796ef4 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestTagMetaService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestTagMetaService.java @@ -18,9 +18,18 @@ */ package com.datastrato.gravitino.storage.relational.service; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.MetadataObjects; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.meta.SchemaEntity; +import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.meta.TagEntity; import com.datastrato.gravitino.storage.RandomIdGenerator; import com.datastrato.gravitino.storage.relational.TestJDBCBackend; @@ -297,4 +306,356 @@ public void testDeleteMetalake() throws IOException { NoSuchEntityException.class, () -> tagMetaService.getTagByIdentifier(TagManager.ofTagIdent(metalakeName + "1", "tag2"))); } + + @Test + public void testAssociateAndDisassociateTagsWithMetadataObject() throws IOException { + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog1", auditInfo); + backend.insert(catalog, false); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of(metalakeName, catalog.name()), + "schema1", + auditInfo); + backend.insert(schema, false); + + TableEntity table = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of(metalakeName, catalog.name(), schema.name()), + "table1", + auditInfo); + backend.insert(table, false); + + // Create tags to associate + TagMetaService tagMetaService = TagMetaService.getInstance(); + TagEntity tagEntity1 = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag1") + .withNamespace(TagManager.ofTagNamespace(metalakeName)) + .withComment("comment") + .withProperties(props) + .withAuditInfo(auditInfo) + .build(); + tagMetaService.insertTag(tagEntity1, false); + + TagEntity tagEntity2 = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag2") + .withNamespace(TagManager.ofTagNamespace(metalakeName)) + .withComment("comment") + .withProperties(props) + .withAuditInfo(auditInfo) + .build(); + tagMetaService.insertTag(tagEntity2, false); + + TagEntity tagEntity3 = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag3") + .withNamespace(TagManager.ofTagNamespace(metalakeName)) + .withComment("comment") + .withProperties(props) + .withAuditInfo(auditInfo) + .build(); + tagMetaService.insertTag(tagEntity3, false); + + // Test associate tags with metadata object + NameIdentifier[] tagsToAdd = + new NameIdentifier[] { + TagManager.ofTagIdent(metalakeName, "tag1"), + TagManager.ofTagIdent(metalakeName, "tag2"), + TagManager.ofTagIdent(metalakeName, "tag3") + }; + + List tagEntities = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToAdd, new NameIdentifier[0]); + Assertions.assertEquals(3, tagEntities.size()); + Assertions.assertTrue(tagEntities.contains(tagEntity1)); + Assertions.assertTrue(tagEntities.contains(tagEntity2)); + Assertions.assertTrue(tagEntities.contains(tagEntity3)); + + // Test disassociate tags with metadata object + NameIdentifier[] tagsToRemove = + new NameIdentifier[] {TagManager.ofTagIdent(metalakeName, "tag1")}; + + List tagEntities1 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), new NameIdentifier[0], tagsToRemove); + + Assertions.assertEquals(2, tagEntities1.size()); + Assertions.assertFalse(tagEntities1.contains(tagEntity1)); + Assertions.assertTrue(tagEntities1.contains(tagEntity2)); + Assertions.assertTrue(tagEntities1.contains(tagEntity3)); + + // Test no tags to associate and disassociate + List tagEntities2 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), new NameIdentifier[0], new NameIdentifier[0]); + Assertions.assertEquals(2, tagEntities2.size()); + Assertions.assertFalse(tagEntities2.contains(tagEntity1)); + Assertions.assertTrue(tagEntities2.contains(tagEntity2)); + Assertions.assertTrue(tagEntities2.contains(tagEntity3)); + + // Test associate and disassociate same tags with metadata object + List tagEntities3 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToRemove, tagsToRemove); + + Assertions.assertEquals(2, tagEntities3.size()); + Assertions.assertFalse(tagEntities3.contains(tagEntity1)); + Assertions.assertTrue(tagEntities3.contains(tagEntity2)); + Assertions.assertTrue(tagEntities3.contains(tagEntity3)); + + // Test associate and disassociate in-existent tags with metadata object + NameIdentifier[] tagsToAdd1 = + new NameIdentifier[] { + TagManager.ofTagIdent(metalakeName, "tag4"), TagManager.ofTagIdent(metalakeName, "tag5") + }; + + NameIdentifier[] tagsToRemove1 = + new NameIdentifier[] { + TagManager.ofTagIdent(metalakeName, "tag6"), TagManager.ofTagIdent(metalakeName, "tag7") + }; + + List tagEntities4 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToAdd1, tagsToRemove1); + + Assertions.assertEquals(2, tagEntities4.size()); + Assertions.assertTrue(tagEntities4.contains(tagEntity2)); + Assertions.assertTrue(tagEntities4.contains(tagEntity3)); + + // Test associate already associated tags with metadata object + Assertions.assertThrows( + EntityAlreadyExistsException.class, + () -> + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToAdd, new NameIdentifier[0])); + + // Test disassociate already disassociated tags with metadata object + List tagEntities5 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), new NameIdentifier[0], tagsToRemove); + + Assertions.assertEquals(2, tagEntities5.size()); + Assertions.assertTrue(tagEntities5.contains(tagEntity2)); + Assertions.assertTrue(tagEntities5.contains(tagEntity3)); + + // Test associate and disassociate with invalid metadata object + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.associateTagsWithMetadataObject( + NameIdentifier.of(metalakeName, "non-existent-catalog"), + catalog.type(), + tagsToAdd, + tagsToRemove)); + + // Test associate and disassociate to a schema + List tagEntities6 = + tagMetaService.associateTagsWithMetadataObject( + schema.nameIdentifier(), schema.type(), tagsToAdd, tagsToRemove); + + Assertions.assertEquals(2, tagEntities6.size()); + Assertions.assertTrue(tagEntities6.contains(tagEntity2)); + Assertions.assertTrue(tagEntities6.contains(tagEntity3)); + + // Test associate and disassociate to a table + List tagEntities7 = + tagMetaService.associateTagsWithMetadataObject( + table.nameIdentifier(), table.type(), tagsToAdd, tagsToRemove); + + Assertions.assertEquals(2, tagEntities7.size()); + Assertions.assertTrue(tagEntities7.contains(tagEntity2)); + Assertions.assertTrue(tagEntities7.contains(tagEntity3)); + } + + @Test + public void testListTagsForMetadataObject() throws IOException { + testAssociateAndDisassociateTagsWithMetadataObject(); + + TagMetaService tagMetaService = TagMetaService.getInstance(); + + // Test list tags for catalog + List tagEntities = + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1"), Entity.EntityType.CATALOG); + Assertions.assertEquals(2, tagEntities.size()); + Assertions.assertTrue( + tagEntities.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag2"))); + Assertions.assertTrue( + tagEntities.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag3"))); + + // Test list tags for schema + List tagEntities1 = + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1"), Entity.EntityType.SCHEMA); + + Assertions.assertEquals(2, tagEntities1.size()); + Assertions.assertTrue( + tagEntities1.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag2"))); + Assertions.assertTrue( + tagEntities1.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag3"))); + + // Test list tags for table + List tagEntities2 = + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE); + + Assertions.assertEquals(2, tagEntities2.size()); + Assertions.assertTrue( + tagEntities2.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag2"))); + Assertions.assertTrue( + tagEntities2.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag3"))); + + // Test list tags for non-existent metadata object + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table2"), + Entity.EntityType.TABLE)); + } + + @Test + public void testGetTagForMetadataObject() throws IOException { + testAssociateAndDisassociateTagsWithMetadataObject(); + + TagMetaService tagMetaService = TagMetaService.getInstance(); + + // Test get tag for catalog + TagEntity tagEntity = + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1"), + Entity.EntityType.CATALOG, + TagManager.ofTagIdent(metalakeName, "tag2")); + Assertions.assertEquals("tag2", tagEntity.name()); + + // Test get tag for schema + TagEntity tagEntity1 = + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1"), + Entity.EntityType.SCHEMA, + TagManager.ofTagIdent(metalakeName, "tag3")); + Assertions.assertEquals("tag3", tagEntity1.name()); + + // Test get tag for table + TagEntity tagEntity2 = + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE, + TagManager.ofTagIdent(metalakeName, "tag2")); + Assertions.assertEquals("tag2", tagEntity2.name()); + + // Test get tag for non-existent metadata object + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table2"), + Entity.EntityType.TABLE, + TagManager.ofTagIdent(metalakeName, "tag2"))); + + // Test get tag for non-existent tag + Throwable e = + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE, + TagManager.ofTagIdent(metalakeName, "tag4"))); + Assertions.assertTrue(e.getMessage().contains("No such tag entity: tag4")); + } + + @Test + public void testListAssociatedMetadataObjectsForTag() throws IOException { + testAssociateAndDisassociateTagsWithMetadataObject(); + + TagMetaService tagMetaService = TagMetaService.getInstance(); + + // Test list associated metadata objects for tag2 + List metadataObjects = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(3, metadataObjects.size()); + Assertions.assertTrue( + metadataObjects.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + Assertions.assertTrue( + metadataObjects.contains( + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA))); + Assertions.assertTrue( + metadataObjects.contains( + MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE))); + + // Test list associated metadata objects for tag3 + List metadataObjects1 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag3")); + + Assertions.assertEquals(3, metadataObjects1.size()); + Assertions.assertTrue( + metadataObjects1.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + Assertions.assertTrue( + metadataObjects1.contains( + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA))); + Assertions.assertTrue( + metadataObjects1.contains( + MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE))); + + // Test list associated metadata objects for non-existent tag + List metadataObjects2 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag4")); + Assertions.assertEquals(0, metadataObjects2.size()); + + // Test metadata object non-exist scenario. + backend.delete( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE, + false); + + List metadataObjects3 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(2, metadataObjects3.size()); + Assertions.assertTrue( + metadataObjects3.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + Assertions.assertTrue( + metadataObjects3.contains( + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA))); + + backend.delete( + NameIdentifier.of(metalakeName, "catalog1", "schema1"), Entity.EntityType.SCHEMA, false); + + List metadataObjects4 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(1, metadataObjects4.size()); + Assertions.assertTrue( + metadataObjects4.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + + backend.delete(NameIdentifier.of(metalakeName, "catalog1"), Entity.EntityType.CATALOG, false); + + List metadataObjects5 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(0, metadataObjects5.size()); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java index 1f36b242c71..62782fe0f1b 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java @@ -24,6 +24,7 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.meta.AuditInfo; @@ -41,6 +42,7 @@ import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.datastrato.gravitino.storage.relational.po.SchemaPO; import com.datastrato.gravitino.storage.relational.po.TablePO; +import com.datastrato.gravitino.storage.relational.po.TagMetadataObjectRelPO; import com.datastrato.gravitino.storage.relational.po.TagPO; import com.datastrato.gravitino.storage.relational.po.TopicPO; import com.datastrato.gravitino.utils.NamespaceUtil; @@ -653,6 +655,21 @@ public void testUpdateTagPOVersion() { assertEquals("this is test2", updatePO.getComment()); } + @Test + public void testTagMetadataObjectRelPO() { + TagMetadataObjectRelPO tagMetadataObjectRelPO = + POConverters.initializeTagMetadataObjectRelPOWithVersion( + 1L, 1L, MetadataObject.Type.CATALOG.toString()); + assertEquals(1L, tagMetadataObjectRelPO.getTagId()); + assertEquals(1L, tagMetadataObjectRelPO.getMetadataObjectId()); + assertEquals( + MetadataObject.Type.CATALOG.toString(), tagMetadataObjectRelPO.getMetadataObjectType()); + + assertEquals(1, tagMetadataObjectRelPO.getCurrentVersion()); + assertEquals(1, tagMetadataObjectRelPO.getLastVersion()); + assertEquals(0, tagMetadataObjectRelPO.getDeletedAt()); + } + private static BaseMetalake createMetalake(Long id, String name, String comment) { AuditInfo auditInfo = AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); diff --git a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java index 9845df8f956..ca56ff59d2b 100644 --- a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java +++ b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java @@ -31,19 +31,29 @@ import static com.datastrato.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; import static com.datastrato.gravitino.Configs.VERSION_RETENTION_COUNT; +import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.GravitinoEnv; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchTagException; +import com.datastrato.gravitino.exceptions.NotFoundException; +import com.datastrato.gravitino.exceptions.TagAlreadyAssociatedException; import com.datastrato.gravitino.exceptions.TagAlreadyExistsException; import com.datastrato.gravitino.lock.LockManager; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.SchemaVersion; +import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.utils.NameIdentifierUtil; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.io.File; @@ -73,6 +83,12 @@ public class TestTagManager { private static final String METALAKE = "metalake_for_tag_test"; + private static final String CATALOG = "catalog_for_tag_test"; + + private static final String SCHEMA = "schema_for_tag_test"; + + private static final String TABLE = "table_for_tag_test"; + private static EntityStore entityStore; private static IdGenerator idGenerator; @@ -115,6 +131,37 @@ public static void setUp() throws IOException, IllegalAccessException { .build(); entityStore.put(metalake, false /* overwritten */); + CatalogEntity catalog = + CatalogEntity.builder() + .withId(idGenerator.nextId()) + .withName(CATALOG) + .withNamespace(Namespace.of(METALAKE)) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment("Test catalog") + .withAuditInfo(audit) + .build(); + entityStore.put(catalog, false /* overwritten */); + + SchemaEntity schema = + SchemaEntity.builder() + .withId(idGenerator.nextId()) + .withName(SCHEMA) + .withNamespace(Namespace.of(METALAKE, CATALOG)) + .withComment("Test schema") + .withAuditInfo(audit) + .build(); + entityStore.put(schema, false /* overwritten */); + + TableEntity table = + TableEntity.builder() + .withId(idGenerator.nextId()) + .withName(TABLE) + .withNamespace(Namespace.of(METALAKE, CATALOG, SCHEMA)) + .withAuditInfo(audit) + .build(); + entityStore.put(table, false /* overwritten */); + tagManager = new TagManager(idGenerator, entityStore); } @@ -130,6 +177,24 @@ public static void tearDown() throws IOException { @AfterEach public void cleanUp() { + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + String[] catalogTags = tagManager.listTagsForMetadataObject(METALAKE, catalogObject); + tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, null, catalogTags); + + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + String[] schemaTags = tagManager.listTagsForMetadataObject(METALAKE, schemaObject); + tagManager.associateTagsForMetadataObject(METALAKE, schemaObject, null, schemaTags); + + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + String[] tableTags = tagManager.listTagsForMetadataObject(METALAKE, tableObject); + tagManager.associateTagsForMetadataObject(METALAKE, tableObject, null, tableTags); + Arrays.stream(tagManager.listTags(METALAKE)).forEach(n -> tagManager.deleteTag(METALAKE, n)); } @@ -245,4 +310,275 @@ public void testDeleteTag() { () -> tagManager.deleteTag("non_existent_metalake", "tag1")); Assertions.assertEquals("Metalake non_existent_metalake does not exist", e.getMessage()); } + + @Test + public void testAssociateTagsForMetadataObject() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + // Test associate tags for catalog + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + String[] tagsToAdd = new String[] {tag1.name(), tag2.name(), tag3.name()}; + + String[] tags = + tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, tagsToAdd, null); + + Assertions.assertEquals(3, tags.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags)); + + // Test disassociate tags for catalog + String[] tagsToRemove = new String[] {tag1.name()}; + String[] tags1 = + tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, null, tagsToRemove); + + Assertions.assertEquals(2, tags1.length); + Assertions.assertEquals(ImmutableSet.of("tag2", "tag3"), ImmutableSet.copyOf(tags1)); + + // Test associate and disassociate no tags for catalog + String[] tags2 = tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, null, null); + + Assertions.assertEquals(2, tags2.length); + Assertions.assertEquals(ImmutableSet.of("tag2", "tag3"), ImmutableSet.copyOf(tags2)); + + // Test re-associate tags for catalog + Throwable e = + Assertions.assertThrows( + TagAlreadyAssociatedException.class, + () -> + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, tagsToAdd, null)); + Assertions.assertTrue(e.getMessage().contains("Failed to associate tags for metadata object")); + + // Test associate and disassociate non-existent tags for catalog + String[] tags3 = + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {"tag4", "tag5"}, new String[] {"tag6"}); + + Assertions.assertEquals(2, tags3.length); + Assertions.assertEquals(ImmutableSet.of("tag2", "tag3"), ImmutableSet.copyOf(tags3)); + + // Test associate tags for non-existent metadata object + MetadataObject nonExistentObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, "non_existent_catalog"), + Entity.EntityType.CATALOG); + Throwable e1 = + Assertions.assertThrows( + NotFoundException.class, + () -> + tagManager.associateTagsForMetadataObject( + METALAKE, nonExistentObject, tagsToAdd, null)); + Assertions.assertTrue(e1.getMessage().contains("Failed to associate tags for metadata object")); + + // Test associate tags for unsupported metadata object + MetadataObject metalakeObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofMetalake(METALAKE), Entity.EntityType.METALAKE); + Throwable e2 = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + tagManager.associateTagsForMetadataObject( + METALAKE, metalakeObject, tagsToAdd, null)); + Assertions.assertTrue( + e2.getMessage().contains("Cannot associate tags for unsupported metadata object type")); + + // Test associate tags for schema + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + String[] tags4 = + tagManager.associateTagsForMetadataObject(METALAKE, schemaObject, tagsToAdd, null); + + Assertions.assertEquals(3, tags4.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags4)); + + // Test associate tags for table + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + String[] tags5 = + tagManager.associateTagsForMetadataObject(METALAKE, tableObject, tagsToAdd, null); + + Assertions.assertEquals(3, tags5.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags5)); + } + + @Test + public void testListMetadataObjectsForTag() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, schemaObject, new String[] {tag1.name(), tag2.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, tableObject, new String[] {tag1.name()}, null); + + MetadataObject[] objects = tagManager.listMetadataObjectsForTag(METALAKE, tag1.name()); + Assertions.assertEquals(3, objects.length); + Assertions.assertEquals( + ImmutableSet.of(catalogObject, schemaObject, tableObject), ImmutableSet.copyOf(objects)); + + MetadataObject[] objects1 = tagManager.listMetadataObjectsForTag(METALAKE, tag2.name()); + Assertions.assertEquals(2, objects1.length); + Assertions.assertEquals( + ImmutableSet.of(catalogObject, schemaObject), ImmutableSet.copyOf(objects1)); + + MetadataObject[] objects2 = tagManager.listMetadataObjectsForTag(METALAKE, tag3.name()); + Assertions.assertEquals(1, objects2.length); + Assertions.assertEquals(ImmutableSet.of(catalogObject), ImmutableSet.copyOf(objects2)); + + // List metadata objects for non-existent tag + Throwable e = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.listMetadataObjectsForTag(METALAKE, "non_existent_tag")); + Assertions.assertTrue( + e.getMessage() + .contains( + "Tag with name non_existent_tag under metalake " + METALAKE + " does not exist")); + } + + @Test + public void testListTagsForMetadataObject() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, schemaObject, new String[] {tag1.name(), tag2.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, tableObject, new String[] {tag1.name()}, null); + + String[] tags = tagManager.listTagsForMetadataObject(METALAKE, catalogObject); + Assertions.assertEquals(3, tags.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags)); + + Tag[] tagsInfo = tagManager.listTagsInfoForMetadataObject(METALAKE, catalogObject); + Assertions.assertEquals(3, tagsInfo.length); + Assertions.assertEquals(ImmutableSet.of(tag1, tag2, tag3), ImmutableSet.copyOf(tagsInfo)); + + String[] tags1 = tagManager.listTagsForMetadataObject(METALAKE, schemaObject); + Assertions.assertEquals(2, tags1.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2"), ImmutableSet.copyOf(tags1)); + + Tag[] tagsInfo1 = tagManager.listTagsInfoForMetadataObject(METALAKE, schemaObject); + Assertions.assertEquals(2, tagsInfo1.length); + Assertions.assertEquals(ImmutableSet.of(tag1, tag2), ImmutableSet.copyOf(tagsInfo1)); + + String[] tags2 = tagManager.listTagsForMetadataObject(METALAKE, tableObject); + Assertions.assertEquals(1, tags2.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tags2)); + + Tag[] tagsInfo2 = tagManager.listTagsInfoForMetadataObject(METALAKE, tableObject); + Assertions.assertEquals(1, tagsInfo2.length); + Assertions.assertEquals(ImmutableSet.of(tag1), ImmutableSet.copyOf(tagsInfo2)); + + // List tags for non-existent metadata object + MetadataObject nonExistentObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, "non_existent_catalog"), + Entity.EntityType.CATALOG); + Throwable e = + Assertions.assertThrows( + NotFoundException.class, + () -> tagManager.listTagsForMetadataObject(METALAKE, nonExistentObject)); + Assertions.assertTrue( + e.getMessage().contains("Failed to list tags for metadata object " + nonExistentObject)); + } + + @Test + public void testGetTagForMetadataObject() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, schemaObject, new String[] {tag1.name(), tag2.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, tableObject, new String[] {tag1.name()}, null); + + Tag result = tagManager.getTagForMetadataObject(METALAKE, catalogObject, tag1.name()); + Assertions.assertEquals(tag1, result); + + Tag result1 = tagManager.getTagForMetadataObject(METALAKE, schemaObject, tag1.name()); + Assertions.assertEquals(tag1, result1); + + Tag result2 = tagManager.getTagForMetadataObject(METALAKE, schemaObject, tag2.name()); + Assertions.assertEquals(tag2, result2); + + Tag result3 = tagManager.getTagForMetadataObject(METALAKE, catalogObject, tag3.name()); + Assertions.assertEquals(tag3, result3); + + // Test get non-existent tag for metadata object + Throwable e = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, catalogObject, "non_existent_tag")); + Assertions.assertTrue(e.getMessage().contains("Tag non_existent_tag does not exist")); + + Throwable e1 = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, schemaObject, tag3.name())); + Assertions.assertTrue(e1.getMessage().contains("Tag tag3 does not exist")); + + Throwable e2 = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, tableObject, tag2.name())); + Assertions.assertTrue(e2.getMessage().contains("Tag tag2 does not exist")); + + // Test get tag for non-existent metadata object + MetadataObject nonExistentObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, "non_existent_catalog"), + Entity.EntityType.CATALOG); + Throwable e3 = + Assertions.assertThrows( + NotFoundException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, nonExistentObject, tag1.name())); + Assertions.assertTrue( + e3.getMessage().contains("Failed to get tag for metadata object " + nonExistentObject)); + } } diff --git a/core/src/test/java/com/datastrato/gravitino/utils/TestNameIdentifierUtil.java b/core/src/test/java/com/datastrato/gravitino/utils/TestNameIdentifierUtil.java index 6d1b68df7f0..d1a53807263 100644 --- a/core/src/test/java/com/datastrato/gravitino/utils/TestNameIdentifierUtil.java +++ b/core/src/test/java/com/datastrato/gravitino/utils/TestNameIdentifierUtil.java @@ -18,9 +18,13 @@ */ package com.datastrato.gravitino.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.MetadataObjects; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException; import com.datastrato.gravitino.exceptions.IllegalNamespaceException; @@ -58,4 +62,66 @@ public void testCheckNameIdentifier() { assertThrows(IllegalNamespaceException.class, () -> NameIdentifierUtil.checkTable(abc)); assertTrue(excep3.getMessage().contains("Table namespace must be non-null and have 3 levels")); } + + @Test + public void testToMetadataObject() { + // test metalake + NameIdentifier metalake = NameIdentifier.of("metalake1"); + MetadataObject metalakeObject = + MetadataObjects.parse("metalake1", MetadataObject.Type.METALAKE); + assertEquals( + metalakeObject, NameIdentifierUtil.toMetadataObject(metalake, Entity.EntityType.METALAKE)); + + // test catalog + NameIdentifier catalog = NameIdentifier.of("metalake1", "catalog1"); + MetadataObject catalogObject = MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG); + assertEquals( + catalogObject, NameIdentifierUtil.toMetadataObject(catalog, Entity.EntityType.CATALOG)); + + // test schema + NameIdentifier schema = NameIdentifier.of("metalake1", "catalog1", "schema1"); + MetadataObject schemaObject = + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA); + assertEquals( + schemaObject, NameIdentifierUtil.toMetadataObject(schema, Entity.EntityType.SCHEMA)); + + // test table + NameIdentifier table = NameIdentifier.of("metalake1", "catalog1", "schema1", "table1"); + MetadataObject tableObject = + MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE); + assertEquals(tableObject, NameIdentifierUtil.toMetadataObject(table, Entity.EntityType.TABLE)); + + // test topic + NameIdentifier topic = NameIdentifier.of("metalake1", "catalog1", "schema1", "topic1"); + MetadataObject topicObject = + MetadataObjects.parse("catalog1.schema1.topic1", MetadataObject.Type.TOPIC); + assertEquals(topicObject, NameIdentifierUtil.toMetadataObject(topic, Entity.EntityType.TOPIC)); + + // test fileset + NameIdentifier fileset = NameIdentifier.of("metalake1", "catalog1", "schema1", "fileset1"); + MetadataObject filesetObject = + MetadataObjects.parse("catalog1.schema1.fileset1", MetadataObject.Type.FILESET); + assertEquals( + filesetObject, NameIdentifierUtil.toMetadataObject(fileset, Entity.EntityType.FILESET)); + + // test column + Throwable e = + assertThrows( + IllegalArgumentException.class, + () -> NameIdentifierUtil.toMetadataObject(fileset, Entity.EntityType.COLUMN)); + assertTrue(e.getMessage().contains("Entity type COLUMN is not supported")); + + // test null + Throwable e1 = + assertThrows( + IllegalArgumentException.class, () -> NameIdentifierUtil.toMetadataObject(null, null)); + assertTrue(e1.getMessage().contains("The identifier and entity type must not be null")); + + // test unsupported + Throwable e2 = + assertThrows( + IllegalArgumentException.class, + () -> NameIdentifierUtil.toMetadataObject(fileset, Entity.EntityType.TAG)); + assertTrue(e2.getMessage().contains("Entity type TAG is not supported")); + } } From bbb7ab3209811f07e7c5c3e838d3a3f092e8965a Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 9 Jul 2024 14:20:34 +0800 Subject: [PATCH 08/27] Fix the style issue --- .../datastrato/gravitino/storage/relational/TestJDBCBackend.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index 9e825eb115f..233bcca869c 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -58,7 +58,6 @@ import com.datastrato.gravitino.storage.relational.mapper.GroupMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.UserMetaMapper; import com.datastrato.gravitino.storage.relational.service.RoleMetaService; -import com.datastrato.gravitino.storage.relational.service.TagMetaService; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import com.datastrato.gravitino.storage.relational.utils.SessionUtils; import com.datastrato.gravitino.tag.TagManager; From 67a4ac04b35537ea622cf901eb6ceff0101985a3 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 10 Jul 2024 10:50:41 +0800 Subject: [PATCH 09/27] Fix some issues --- .../com/datastrato/gravitino/SupportsExtraOperations.java | 8 +++++++- .../datastrato/gravitino/server/TestGravitinoServer.java | 5 ++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java b/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java index 5b62abb6fac..0d00535a844 100644 --- a/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java +++ b/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java @@ -36,6 +36,7 @@ public interface SupportsExtraOperations { * * @param tagIdent The identifier of the tag. * @return The list of metadata objects associated with the given tag. + * @throws IOException If an error occurs while accessing the entity store. */ MetadataObject[] listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) throws IOException; @@ -46,6 +47,7 @@ public interface SupportsExtraOperations { * @param objectType The type of the metadata object. * @return The list of tags associated with the given metadata object. * @throws NoSuchEntityException if the metadata object does not exist. + * @throws IOException If an error occurs while accessing the entity store. */ TagEntity[] listAssociatedTagsForMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType) @@ -58,7 +60,9 @@ TagEntity[] listAssociatedTagsForMetadataObject( * @param objectType The type of the metadata object. * @param tagIdent The identifier of the tag. * @return The tag associated with the metadata object. - * @throws NoSuchEntityException if the metadata object does not exist. + * @throws NoSuchEntityException if the metadata object does not exist or the tag is not + * associated to the metadata object. + * @throws IOException If an error occurs while accessing the entity store. */ TagEntity getTagForMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) @@ -73,6 +77,8 @@ TagEntity getTagForMetadataObject( * @param tagsToRemove the name of tags to remove from the metadata object. * @return The list of tags associated with the metadata object after the operation. * @throws NoSuchEntityException if the metadata object does not exist. + * @throws EntityAlreadyExistsException if tags already associated with the metadata object. + * @throws IOException If an error occurs while accessing the entity store. */ TagEntity[] associateTagsWithMetadataObject( NameIdentifier objectIdent, diff --git a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java index 1c2f3c58168..e54909aa4d7 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java +++ b/server/src/test/java/com/datastrato/gravitino/server/TestGravitinoServer.java @@ -36,7 +36,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -100,7 +99,7 @@ public void tearDown() { } } - @Disabled("Disabled until shifting to use relational entity store as default storage") + @Test public void testInitialize() { gravitinoServer.initialize(); } @@ -111,7 +110,7 @@ void testConfig() { ROCKS_DB_STORE_PATH, spyServerConfig.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); } - @Disabled("Disabled until shifting to use relational entity store as default storage") + @Test public void testStartAndStop() throws Exception { gravitinoServer.initialize(); gravitinoServer.start(); From eaa2081c45311c80740871afa7d8ae5198b69c2a Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Fri, 12 Jul 2024 15:28:24 +0800 Subject: [PATCH 10/27] Address the comments --- .../com/datastrato/gravitino/EntityStore.java | 7 +-- .../storage/relational/JDBCBackend.java | 17 +++---- .../storage/relational/RelationalBackend.java | 4 +- .../relational/RelationalEntityStore.java | 13 ++--- .../mapper/TagMetadataObjectRelMapper.java | 26 +++++----- .../relational/po/TagMetadataObjectRelPO.java | 17 +++---- .../storage/relational/po/TagPO.java | 21 +++++---- .../storage/relational/po/TopicPO.java | 26 +++++----- .../MetadataObjectService.java} | 14 ++---- .../relational/service/RoleMetaService.java | 6 +-- .../relational/service/TagMetaService.java | 9 ++-- .../SupportsTagOperations.java} | 19 +++++--- .../datastrato/gravitino/tag/TagManager.java | 47 ++++++++++--------- .../gravitino/utils/MetadataObjectUtil.java | 38 +++++++-------- scripts/h2/schema-h2.sql | 2 +- scripts/mysql/schema-0.6.0-mysql.sql | 2 +- .../mysql/upgrade-0.5.0-to-0.6.0-mysql.sql | 2 +- 17 files changed, 135 insertions(+), 135 deletions(-) rename core/src/main/java/com/datastrato/gravitino/storage/relational/{utils/MetadataObjectUtils.java => service/MetadataObjectService.java} (89%) rename core/src/main/java/com/datastrato/gravitino/{SupportsExtraOperations.java => tag/SupportsTagOperations.java} (84%) diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStore.java b/core/src/main/java/com/datastrato/gravitino/EntityStore.java index 750aeee849f..d6d7971aeb9 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStore.java @@ -20,6 +20,7 @@ import com.datastrato.gravitino.Entity.EntityType; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.tag.SupportsTagOperations; import com.datastrato.gravitino.utils.Executable; import java.io.Closeable; import java.io.IOException; @@ -183,12 +184,12 @@ R executeInTransaction(Executable executable) throws E, IOException; /** - * Get the extra operations that are supported by the entity store. + * Get the extra tag operations that are supported by the entity store. * - * @return the extra operations object that are supported by the entity store + * @return the tag operations object that are supported by the entity store * @throws UnsupportedOperationException if the extra operations are not supported */ - default SupportsExtraOperations extraOperations() { + default SupportsTagOperations tagOperations() { throw new UnsupportedOperationException("extra operations are not supported"); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index d4c701a569b..a2a41e7da60 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -330,20 +330,16 @@ public void close() throws IOException { } @Override - public MetadataObject[] listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + public List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) throws IOException { - return TagMetaService.getInstance() - .listAssociatedMetadataObjectIdentsForTag(tagIdent) - .toArray(new MetadataObject[0]); + return TagMetaService.getInstance().listAssociatedMetadataObjectIdentsForTag(tagIdent); } @Override - public TagEntity[] listAssociatedTagsForMetadataObject( + public List listAssociatedTagsForMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType) throws NoSuchEntityException, IOException { - return TagMetaService.getInstance() - .listTagsForMetadataObject(objectIdent, objectType) - .toArray(new TagEntity[0]); + return TagMetaService.getInstance().listTagsForMetadataObject(objectIdent, objectType); } @Override @@ -354,15 +350,14 @@ public TagEntity getTagForMetadataObject( } @Override - public TagEntity[] associateTagsWithMetadataObject( + public List associateTagsWithMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier[] tagsToAdd, NameIdentifier[] tagsToRemove) throws NoSuchEntityException, EntityAlreadyExistsException, IOException { return TagMetaService.getInstance() - .associateTagsWithMetadataObject(objectIdent, objectType, tagsToAdd, tagsToRemove) - .toArray(new TagEntity[0]); + .associateTagsWithMetadataObject(objectIdent, objectType, tagsToAdd, tagsToRemove); } enum JDBCBackendType { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java index bb53827eeb0..d6217fff34b 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalBackend.java @@ -24,15 +24,15 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.SupportsExtraOperations; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.tag.SupportsTagOperations; import java.io.Closeable; import java.io.IOException; import java.util.List; import java.util.function.Function; /** Interface defining the operations for a Relation Backend. */ -public interface RelationalBackend extends Closeable, SupportsExtraOperations { +public interface RelationalBackend extends Closeable, SupportsTagOperations { /** * Initializes the Relational Backend environment with the provided configuration. diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java index 9e9f70989ec..6774a521f2c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/RelationalEntityStore.java @@ -30,9 +30,9 @@ import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.SupportsExtraOperations; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.TagEntity; +import com.datastrato.gravitino.tag.SupportsTagOperations; import com.datastrato.gravitino.utils.Executable; import com.google.common.collect.ImmutableMap; import java.io.IOException; @@ -46,7 +46,7 @@ * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link * RelationalBackend} interface */ -public class RelationalEntityStore implements EntityStore, SupportsExtraOperations { +public class RelationalEntityStore implements EntityStore, SupportsTagOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); public static final ImmutableMap RELATIONAL_BACKENDS = ImmutableMap.of( @@ -136,18 +136,19 @@ public void close() throws IOException { backend.close(); } - public SupportsExtraOperations extraOperations() { + @Override + public SupportsTagOperations tagOperations() { return this; } @Override - public MetadataObject[] listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + public List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) throws IOException { return backend.listAssociatedMetadataObjectsForTag(tagIdent); } @Override - public TagEntity[] listAssociatedTagsForMetadataObject( + public List listAssociatedTagsForMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType) throws NoSuchEntityException, IOException { return backend.listAssociatedTagsForMetadataObject(objectIdent, objectType); @@ -161,7 +162,7 @@ public TagEntity getTagForMetadataObject( } @Override - public TagEntity[] associateTagsWithMetadataObject( + public List associateTagsWithMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier[] tagsToAdd, diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index 70d644b9e48..cbdf339fc2d 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -70,19 +70,19 @@ TagPO getTagPOsByMetadataObjectAndTagName( @Param("tagName") String tagName); @Select( - "SELECT tmo.tag_id as tagId, tmo.metadata_object_id as metadataObjectId," - + " tmo.metadata_object_type as metadataObjectType, tmo.audit_info as auditInfo," - + " tmo.current_version as currentVersion, tmo.last_version as lastVersion," - + " tmo.deleted_at as deletedAt" + "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," + + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," + + " te.current_version as currentVersion, te.last_version as lastVersion," + + " te.deleted_at as deletedAt" + " FROM " + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " tmo JOIN " + + " te JOIN " + TagMetaMapper.TAG_TABLE_NAME + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME - + " mm ON tmo.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" + + " mm ON te.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" - + " AND tmo.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0") + + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0") List listTagMetadataObjectRelsByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); @@ -128,26 +128,26 @@ void batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( @Update( "UPDATE " + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " tmo SET tmo.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE tmo.tag_id IN (SELECT tm.tag_id FROM " + + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " + TagMetaMapper.TAG_TABLE_NAME + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " + MetalakeMetaMapper.TABLE_NAME + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.deleted_at = 0) AND tmo.deleted_at = 0") + + " AND tm.deleted_at = 0) AND te.deleted_at = 0") Integer softDeleteTagMetadataObjectRelsByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); @Update( "UPDATE " + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " tmo SET tmo.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE EXISTS (SELECT * FROM " + TagMetaMapper.TAG_TABLE_NAME - + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = tmo.tag_id" - + " AND tm.deleted_at = 0) AND tmo.deleted_at = 0") + + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0") void softDeleteTagMetadataObjectRelsByMetalakeId(@Param("metalakeId") Long metalakeId); @Delete( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java index 9122fc50cb3..6637f5189d6 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagMetadataObjectRelPO.java @@ -18,6 +18,7 @@ */ package com.datastrato.gravitino.storage.relational.po; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; import lombok.Getter; import org.apache.commons.lang3.StringUtils; @@ -45,18 +46,18 @@ public boolean equals(Object o) { return false; } TagMetadataObjectRelPO tagRelPO = (TagMetadataObjectRelPO) o; - return java.util.Objects.equals(tagId, tagRelPO.tagId) - && java.util.Objects.equals(metadataObjectId, tagRelPO.metadataObjectId) - && java.util.Objects.equals(metadataObjectType, tagRelPO.metadataObjectType) - && java.util.Objects.equals(auditInfo, tagRelPO.auditInfo) - && java.util.Objects.equals(currentVersion, tagRelPO.currentVersion) - && java.util.Objects.equals(lastVersion, tagRelPO.lastVersion) - && java.util.Objects.equals(deletedAt, tagRelPO.deletedAt); + return Objects.equal(tagId, tagRelPO.tagId) + && Objects.equal(metadataObjectId, tagRelPO.metadataObjectId) + && Objects.equal(metadataObjectType, tagRelPO.metadataObjectType) + && Objects.equal(auditInfo, tagRelPO.auditInfo) + && Objects.equal(currentVersion, tagRelPO.currentVersion) + && Objects.equal(lastVersion, tagRelPO.lastVersion) + && Objects.equal(deletedAt, tagRelPO.deletedAt); } @Override public int hashCode() { - return java.util.Objects.hash( + return Objects.hashCode( tagId, metadataObjectId, metadataObjectType, diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagPO.java index 8924bdfd201..54f9db3edfc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagPO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TagPO.java @@ -18,6 +18,7 @@ */ package com.datastrato.gravitino.storage.relational.po; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; import lombok.Getter; import org.apache.commons.lang3.StringUtils; @@ -47,20 +48,20 @@ public boolean equals(Object o) { return false; } TagPO tagPO = (TagPO) o; - return java.util.Objects.equals(tagId, tagPO.tagId) - && java.util.Objects.equals(tagName, tagPO.tagName) - && java.util.Objects.equals(metalakeId, tagPO.metalakeId) - && java.util.Objects.equals(comment, tagPO.comment) - && java.util.Objects.equals(properties, tagPO.properties) - && java.util.Objects.equals(auditInfo, tagPO.auditInfo) - && java.util.Objects.equals(currentVersion, tagPO.currentVersion) - && java.util.Objects.equals(lastVersion, tagPO.lastVersion) - && java.util.Objects.equals(deletedAt, tagPO.deletedAt); + return Objects.equal(tagId, tagPO.tagId) + && Objects.equal(tagName, tagPO.tagName) + && Objects.equal(metalakeId, tagPO.metalakeId) + && Objects.equal(comment, tagPO.comment) + && Objects.equal(properties, tagPO.properties) + && Objects.equal(auditInfo, tagPO.auditInfo) + && Objects.equal(currentVersion, tagPO.currentVersion) + && Objects.equal(lastVersion, tagPO.lastVersion) + && Objects.equal(deletedAt, tagPO.deletedAt); } @Override public int hashCode() { - return java.util.Objects.hash( + return Objects.hashCode( tagId, tagName, metalakeId, diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TopicPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TopicPO.java index 438c2e2142e..72ad6d08ba9 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TopicPO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/TopicPO.java @@ -18,8 +18,8 @@ */ package com.datastrato.gravitino.storage.relational.po; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; -import java.util.Objects; import lombok.Getter; @Getter @@ -49,22 +49,22 @@ public boolean equals(Object o) { return false; } TopicPO topicPO = (TopicPO) o; - return Objects.equals(topicId, topicPO.topicId) - && Objects.equals(topicName, topicPO.topicName) - && Objects.equals(metalakeId, topicPO.metalakeId) - && Objects.equals(catalogId, topicPO.catalogId) - && Objects.equals(schemaId, topicPO.schemaId) - && Objects.equals(comment, topicPO.comment) - && Objects.equals(properties, topicPO.properties) - && Objects.equals(auditInfo, topicPO.auditInfo) - && Objects.equals(currentVersion, topicPO.currentVersion) - && Objects.equals(lastVersion, topicPO.lastVersion) - && Objects.equals(deletedAt, topicPO.deletedAt); + return Objects.equal(topicId, topicPO.topicId) + && Objects.equal(topicName, topicPO.topicName) + && Objects.equal(metalakeId, topicPO.metalakeId) + && Objects.equal(catalogId, topicPO.catalogId) + && Objects.equal(schemaId, topicPO.schemaId) + && Objects.equal(comment, topicPO.comment) + && Objects.equal(properties, topicPO.properties) + && Objects.equal(auditInfo, topicPO.auditInfo) + && Objects.equal(currentVersion, topicPO.currentVersion) + && Objects.equal(lastVersion, topicPO.lastVersion) + && Objects.equal(deletedAt, topicPO.deletedAt); } @Override public int hashCode() { - return Objects.hash( + return Objects.hashCode( topicId, topicName, metalakeId, diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetadataObjectService.java similarity index 89% rename from core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java rename to core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetadataObjectService.java index 1d802d956ed..66060008758 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetadataObjectService.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.datastrato.gravitino.storage.relational.utils; +package com.datastrato.gravitino.storage.relational.service; import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.MetadataObject; @@ -27,28 +27,22 @@ import com.datastrato.gravitino.storage.relational.po.SchemaPO; import com.datastrato.gravitino.storage.relational.po.TablePO; import com.datastrato.gravitino.storage.relational.po.TopicPO; -import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; -import com.datastrato.gravitino.storage.relational.service.FilesetMetaService; -import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; -import com.datastrato.gravitino.storage.relational.service.SchemaMetaService; -import com.datastrato.gravitino.storage.relational.service.TableMetaService; -import com.datastrato.gravitino.storage.relational.service.TopicMetaService; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import java.util.List; import javax.annotation.Nullable; /** - * MetadataObjectUtils is used for converting full name to entity id and converting entity id to + * MetadataObjectService is used for converting full name to entity id and converting entity id to * full name. */ -public class MetadataObjectUtils { +public class MetadataObjectService { private static final String DOT = "."; private static final Joiner DOT_JOINER = Joiner.on(DOT); private static final Splitter DOT_SPLITTER = Splitter.on(DOT); - private MetadataObjectUtils() {} + private MetadataObjectService() {} public static long getMetadataObjectId( long metalakeId, String fullName, MetadataObject.Type type) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java index c69083168ca..9aaceed4a7e 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java @@ -33,7 +33,6 @@ import com.datastrato.gravitino.storage.relational.po.RolePO; import com.datastrato.gravitino.storage.relational.po.SecurableObjectPO; import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; -import com.datastrato.gravitino.storage.relational.utils.MetadataObjectUtils; import com.datastrato.gravitino.storage.relational.utils.POConverters; import com.datastrato.gravitino.storage.relational.utils.SessionUtils; import com.google.common.collect.Lists; @@ -108,7 +107,8 @@ public void insertRole(RoleEntity roleEntity, boolean overwritten) throws IOExce POConverters.initializeSecurablePOBuilderWithVersion( roleEntity.id(), object, getEntityType(object)); objectBuilder.withEntityId( - MetadataObjectUtils.getMetadataObjectId(metalakeId, object.fullName(), object.type())); + MetadataObjectService.getMetadataObjectId( + metalakeId, object.fullName(), object.type())); securableObjectPOs.add(objectBuilder.build()); } @@ -154,7 +154,7 @@ public RoleEntity getRoleByIdentifier(NameIdentifier identifier) { for (SecurableObjectPO securableObjectPO : securableObjectPOs) { String fullName = - MetadataObjectUtils.getMetadataObjectFullName( + MetadataObjectService.getMetadataObjectFullName( securableObjectPO.getType(), securableObjectPO.getEntityId()); if (fullName != null) { securableObjects.add( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java index 8c6e61a91cd..a79adf53480 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TagMetaService.java @@ -33,7 +33,6 @@ import com.datastrato.gravitino.storage.relational.po.TagMetadataObjectRelPO; import com.datastrato.gravitino.storage.relational.po.TagPO; import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; -import com.datastrato.gravitino.storage.relational.utils.MetadataObjectUtils; import com.datastrato.gravitino.storage.relational.utils.POConverters; import com.datastrato.gravitino.storage.relational.utils.SessionUtils; import com.datastrato.gravitino.tag.TagManager; @@ -165,7 +164,7 @@ public List listTagsForMetadataObject( try { Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); Long metadataObjectId = - MetadataObjectUtils.getMetadataObjectId( + MetadataObjectService.getMetadataObjectId( metalakeId, metadataObject.fullName(), metadataObject.type()); tagPOs = @@ -194,7 +193,7 @@ public TagEntity getTagForMetadataObject( try { Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); Long metadataObjectId = - MetadataObjectUtils.getMetadataObjectId( + MetadataObjectService.getMetadataObjectId( metalakeId, metadataObject.fullName(), metadataObject.type()); tagPO = @@ -233,7 +232,7 @@ public List listAssociatedMetadataObjectIdentsForTag(NameIdentif List metadataObjects = Lists.newArrayList(); for (TagMetadataObjectRelPO po : tagMetadataObjectRelPOs) { String fullName = - MetadataObjectUtils.getMetadataObjectFullName( + MetadataObjectService.getMetadataObjectFullName( po.getMetadataObjectType(), po.getMetadataObjectId()); // Metadata object may be deleted asynchronously when we query the name, so it will return @@ -265,7 +264,7 @@ public List associateTagsWithMetadataObject( try { Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); Long metadataObjectId = - MetadataObjectUtils.getMetadataObjectId( + MetadataObjectService.getMetadataObjectId( metalakeId, metadataObject.fullName(), metadataObject.type()); // Fetch all the tags need to associate with the metadata object. diff --git a/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java b/core/src/main/java/com/datastrato/gravitino/tag/SupportsTagOperations.java similarity index 84% rename from core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java rename to core/src/main/java/com/datastrato/gravitino/tag/SupportsTagOperations.java index 0d00535a844..715ae9c4819 100644 --- a/core/src/main/java/com/datastrato/gravitino/SupportsExtraOperations.java +++ b/core/src/main/java/com/datastrato/gravitino/tag/SupportsTagOperations.java @@ -17,19 +17,25 @@ * under the License. */ -package com.datastrato.gravitino; +package com.datastrato.gravitino.tag; +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.EntityAlreadyExistsException; +import com.datastrato.gravitino.EntityStore; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.TagEntity; import java.io.IOException; +import java.util.List; /** - * An interface to support extra entity store operations, this interface should be mixed with {@link + * An interface to support extra tag operations, this interface should be mixed with {@link * EntityStore} to provide extra operations. * *

Any operations that can be done by the entity store should be added here. */ -public interface SupportsExtraOperations { +public interface SupportsTagOperations { /** * List all the metadata objects that are associated with the given tag. @@ -38,7 +44,8 @@ public interface SupportsExtraOperations { * @return The list of metadata objects associated with the given tag. * @throws IOException If an error occurs while accessing the entity store. */ - MetadataObject[] listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) throws IOException; + List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + throws IOException; /** * List all the tags that are associated with the given metadata object. @@ -49,7 +56,7 @@ public interface SupportsExtraOperations { * @throws NoSuchEntityException if the metadata object does not exist. * @throws IOException If an error occurs while accessing the entity store. */ - TagEntity[] listAssociatedTagsForMetadataObject( + List listAssociatedTagsForMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType) throws NoSuchEntityException, IOException; @@ -80,7 +87,7 @@ TagEntity getTagForMetadataObject( * @throws EntityAlreadyExistsException if tags already associated with the metadata object. * @throws IOException If an error occurs while accessing the entity store. */ - TagEntity[] associateTagsWithMetadataObject( + List associateTagsWithMetadataObject( NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier[] tagsToAdd, diff --git a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java index 675be52e9c7..847ad01e72e 100644 --- a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java +++ b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java @@ -24,7 +24,6 @@ import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.SupportsExtraOperations; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchTagException; @@ -60,7 +59,7 @@ public class TagManager { private final EntityStore entityStore; - private final SupportsExtraOperations supportsExtraOperations; + private final SupportsTagOperations supportsTagOperations; public TagManager(IdGenerator idGenerator, EntityStore entityStore) { if (entityStore instanceof KvEntityStore) { @@ -71,15 +70,15 @@ public TagManager(IdGenerator idGenerator, EntityStore entityStore) { throw new RuntimeException(errorMsg); } - if (!(entityStore instanceof SupportsExtraOperations)) { + if (!(entityStore instanceof SupportsTagOperations)) { String errorMsg = - "TagManager cannot run with entity store that does not support extra operations, " + "TagManager cannot run with entity store that does not support tag operations, " + "please configure the entity store to use relational entity store and restart the Gravitino server"; LOG.error(errorMsg); throw new RuntimeException(errorMsg); } - this.supportsExtraOperations = entityStore.extraOperations(); + this.supportsTagOperations = entityStore.tagOperations(); this.idGenerator = idGenerator; this.entityStore = entityStore; @@ -217,7 +216,9 @@ public MetadataObject[] listMetadataObjectsForTag(String metalake, String name) "Tag with name %s under metalake %s does not exist", name, metalake); } - return supportsExtraOperations.listAssociatedMetadataObjectsForTag(tagId); + return supportsTagOperations + .listAssociatedMetadataObjectsForTag(tagId) + .toArray(new MetadataObject[0]); } catch (IOException e) { LOG.error("Failed to list metadata objects for tag {}", name, e); throw new RuntimeException(e); @@ -242,8 +243,9 @@ public Tag[] listTagsInfoForMetadataObject(String metalake, MetadataObject metad LockType.READ, () -> { try { - return supportsExtraOperations.listAssociatedTagsForMetadataObject( - entityIdent, entityType); + return supportsTagOperations + .listAssociatedTagsForMetadataObject(entityIdent, entityType) + .toArray(new Tag[0]); } catch (NoSuchEntityException e) { throw new NotFoundException( e, "Failed to list tags for metadata object %s due to not found", metadataObject); @@ -265,8 +267,7 @@ public Tag getTagForMetadataObject(String metalake, MetadataObject metadataObjec LockType.READ, () -> { try { - return supportsExtraOperations.getTagForMetadataObject( - entityIdent, entityType, tagIdent); + return supportsTagOperations.getTagForMetadataObject(entityIdent, entityType, tagIdent); } catch (NoSuchEntityException e) { if (e.getMessage().contains("No such tag entity")) { throw new NoSuchTagException( @@ -294,21 +295,20 @@ public String[] associateTagsForMetadataObject( NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + // Remove all the tags that are both set to add and remove Set tagsToAddSet = tagsToAdd == null ? Sets.newHashSet() : Sets.newHashSet(tagsToAdd); - if (tagsToRemove != null) { - for (String tag : tagsToRemove) { - tagsToAddSet.remove(tag); - } - } + Set tagsToRemoveSet = + tagsToRemove == null ? Sets.newHashSet() : Sets.newHashSet(tagsToRemove); + Set common = Sets.intersection(tagsToAddSet, tagsToRemoveSet); + tagsToAddSet.removeAll(common); + tagsToRemoveSet.removeAll(common); NameIdentifier[] tagsToAddIdent = tagsToAddSet.stream().map(tag -> ofTagIdent(metalake, tag)).toArray(NameIdentifier[]::new); NameIdentifier[] tagsToRemoveIdent = - tagsToRemove == null - ? new NameIdentifier[0] - : Sets.newHashSet(tagsToRemove).stream() - .map(tag -> ofTagIdent(metalake, tag)) - .toArray(NameIdentifier[]::new); + tagsToRemoveSet.stream() + .map(tag -> ofTagIdent(metalake, tag)) + .toArray(NameIdentifier[]::new); // TODO. We need to add a write lock to Tag's namespace to avoid tag alteration and deletion // during the association operation. @@ -317,9 +317,10 @@ public String[] associateTagsForMetadataObject( LockType.READ, () -> { try { - return Arrays.stream( - supportsExtraOperations.associateTagsWithMetadataObject( - entityIdent, entityType, tagsToAddIdent, tagsToRemoveIdent)) + return supportsTagOperations + .associateTagsWithMetadataObject( + entityIdent, entityType, tagsToAddIdent, tagsToRemoveIdent) + .stream() .map(Tag::name) .toArray(String[]::new); } catch (NoSuchEntityException e) { diff --git a/core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java b/core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java index fe87b4f1ec5..4c7a11fe97b 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/MetadataObjectUtil.java @@ -22,11 +22,25 @@ import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.NameIdentifier; import com.google.common.base.Joiner; +import com.google.common.collect.BiMap; +import com.google.common.collect.ImmutableBiMap; +import java.util.Optional; public class MetadataObjectUtil { private static final Joiner DOT = Joiner.on("."); + private static final BiMap TYPE_TO_TYPE_MAP = + ImmutableBiMap.builder() + .put(MetadataObject.Type.METALAKE, Entity.EntityType.METALAKE) + .put(MetadataObject.Type.CATALOG, Entity.EntityType.CATALOG) + .put(MetadataObject.Type.SCHEMA, Entity.EntityType.SCHEMA) + .put(MetadataObject.Type.TABLE, Entity.EntityType.TABLE) + .put(MetadataObject.Type.TOPIC, Entity.EntityType.TOPIC) + .put(MetadataObject.Type.FILESET, Entity.EntityType.FILESET) + .put(MetadataObject.Type.COLUMN, Entity.EntityType.COLUMN) + .build(); + private MetadataObjectUtil() {} /** @@ -37,25 +51,11 @@ private MetadataObjectUtil() {} * @throws IllegalArgumentException if the metadata object type is unknown */ public static Entity.EntityType toEntityType(MetadataObject metadataObject) { - switch (metadataObject.type()) { - case METALAKE: - return Entity.EntityType.METALAKE; - case CATALOG: - return Entity.EntityType.CATALOG; - case SCHEMA: - return Entity.EntityType.SCHEMA; - case TABLE: - return Entity.EntityType.TABLE; - case TOPIC: - return Entity.EntityType.TOPIC; - case FILESET: - return Entity.EntityType.FILESET; - case COLUMN: - return Entity.EntityType.COLUMN; - default: - throw new IllegalArgumentException( - "Unknown metadata object type: " + metadataObject.type()); - } + return Optional.ofNullable(TYPE_TO_TYPE_MAP.get(metadataObject.type())) + .orElseThrow( + () -> + new IllegalArgumentException( + "Unknown metadata object type: " + metadataObject.type())); } /** diff --git a/scripts/h2/schema-h2.sql b/scripts/h2/schema-h2.sql index f4c961cb61d..0ebf91fa39a 100644 --- a/scripts/h2/schema-h2.sql +++ b/scripts/h2/schema-h2.sql @@ -241,7 +241,7 @@ CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', PRIMARY KEY (`id`), - UNIQUE KEY `uk_ti_mi_del` (`tag_id`, `metadata_object_id`, `deleted_at`), + UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), KEY `idx_tid` (`tag_id`), KEY `idx_mid` (`metadata_object_id`) ) ENGINE=InnoDB; diff --git a/scripts/mysql/schema-0.6.0-mysql.sql b/scripts/mysql/schema-0.6.0-mysql.sql index 8418a5602aa..b50609da3c4 100644 --- a/scripts/mysql/schema-0.6.0-mysql.sql +++ b/scripts/mysql/schema-0.6.0-mysql.sql @@ -234,7 +234,7 @@ CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', PRIMARY KEY (`id`), - UNIQUE KEY `uk_ti_mi_del` (`tag_id`, `metadata_object_id`, `deleted_at`), + UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), KEY `idx_tid` (`tag_id`), KEY `idx_mid` (`metadata_object_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata object relation'; diff --git a/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql b/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql index 8fb71f73012..3048c67abe3 100644 --- a/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql +++ b/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql @@ -60,7 +60,7 @@ CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', PRIMARY KEY (`id`), - UNIQUE KEY `uk_ti_mi_del` (`tag_id`, `metadata_object_id`, `deleted_at`), + UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), KEY `idx_tid` (`tag_id`), KEY `idx_mid` (`metadata_object_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata object relation'; From bba5b79c5b95a16415eda2841f2b62e99803a5a6 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Fri, 12 Jul 2024 15:40:22 +0800 Subject: [PATCH 11/27] Fix issue and add ut --- .../com/datastrato/gravitino/tag/TagManager.java | 2 +- .../datastrato/gravitino/tag/TestTagManager.java | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java index 847ad01e72e..d83fb9462f5 100644 --- a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java +++ b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java @@ -299,7 +299,7 @@ public String[] associateTagsForMetadataObject( Set tagsToAddSet = tagsToAdd == null ? Sets.newHashSet() : Sets.newHashSet(tagsToAdd); Set tagsToRemoveSet = tagsToRemove == null ? Sets.newHashSet() : Sets.newHashSet(tagsToRemove); - Set common = Sets.intersection(tagsToAddSet, tagsToRemoveSet); + Set common = Sets.intersection(tagsToAddSet, tagsToRemoveSet).immutableCopy(); tagsToAddSet.removeAll(common); tagsToRemoveSet.removeAll(common); diff --git a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java index ca56ff59d2b..f2f74adacdf 100644 --- a/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java +++ b/core/src/test/java/com/datastrato/gravitino/tag/TestTagManager.java @@ -397,14 +397,24 @@ public void testAssociateTagsForMetadataObject() { Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags4)); // Test associate tags for table + String[] tagsToAdd1 = new String[] {tag1.name()}; MetadataObject tableObject = NameIdentifierUtil.toMetadataObject( NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); String[] tags5 = - tagManager.associateTagsForMetadataObject(METALAKE, tableObject, tagsToAdd, null); + tagManager.associateTagsForMetadataObject(METALAKE, tableObject, tagsToAdd1, null); - Assertions.assertEquals(3, tags5.length); - Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags5)); + Assertions.assertEquals(1, tags5.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tags5)); + + // Test associate and disassociate same tags for table + String[] tagsToAdd2 = new String[] {tag2.name(), tag3.name()}; + String[] tagsToRemove1 = new String[] {tag2.name()}; + String[] tags6 = + tagManager.associateTagsForMetadataObject(METALAKE, tableObject, tagsToAdd2, tagsToRemove1); + + Assertions.assertEquals(2, tags6.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag3"), ImmutableSet.copyOf(tags6)); } @Test From 8abab544a8ae725b116487378f432e00c8df510a Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Fri, 12 Jul 2024 16:04:25 +0800 Subject: [PATCH 12/27] Fix style issue --- .../datastrato/gravitino/tag/TagManager.java | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java index d83fb9462f5..1cf78a25c56 100644 --- a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java +++ b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java @@ -315,31 +315,35 @@ public String[] associateTagsForMetadataObject( return TreeLockUtils.doWithTreeLock( entityIdent, LockType.READ, - () -> { - try { - return supportsTagOperations - .associateTagsWithMetadataObject( - entityIdent, entityType, tagsToAddIdent, tagsToRemoveIdent) - .stream() - .map(Tag::name) - .toArray(String[]::new); - } catch (NoSuchEntityException e) { - throw new NotFoundException( - e, - "Failed to associate tags for metadata object %s due to not found", - metadataObject); - } catch (EntityAlreadyExistsException e) { - throw new TagAlreadyAssociatedException( - e, - "Failed to associate tags for metadata object due to some tags %s already " - + "associated to the metadata object %s", - Arrays.toString(tagsToAdd), - metadataObject); - } catch (IOException e) { - LOG.error("Failed to associate tags for metadata object {}", metadataObject, e); - throw new RuntimeException(e); - } - }); + () -> + TreeLockUtils.doWithTreeLock( + NameIdentifier.of(ofTagNamespace(metalake).levels()), + LockType.WRITE, + () -> { + try { + return supportsTagOperations + .associateTagsWithMetadataObject( + entityIdent, entityType, tagsToAddIdent, tagsToRemoveIdent) + .stream() + .map(Tag::name) + .toArray(String[]::new); + } catch (NoSuchEntityException e) { + throw new NotFoundException( + e, + "Failed to associate tags for metadata object %s due to not found", + metadataObject); + } catch (EntityAlreadyExistsException e) { + throw new TagAlreadyAssociatedException( + e, + "Failed to associate tags for metadata object due to some tags %s already " + + "associated to the metadata object %s", + Arrays.toString(tagsToAdd), + metadataObject); + } catch (IOException e) { + LOG.error("Failed to associate tags for metadata object {}", metadataObject, e); + throw new RuntimeException(e); + } + })); } private static void checkMetalakeExists(String metalake, EntityStore entityStore) { From 61276baedd6f5582b92ab61a5ca74a61b17dbde1 Mon Sep 17 00:00:00 2001 From: bknbkn <67318028+bknbkn@users.noreply.github.com> Date: Fri, 12 Jul 2024 16:02:22 +0800 Subject: [PATCH 13/27] [#4112] feat(API, client): Add fileset comment removal operations (#4123) ### What changes were proposed in this pull request? Add fileset commet removal operations: FilesetChange.RemoveComment Fix: #4112 ### Why are the changes needed? Fileset can be created without comment now, when user added a comment to fileset, they should be allowed to retract the comment. Like other FilesetChange, FilesetChange.RemoveComment can be used for **alterFileset()** method ### Does this PR introduce _any_ user-facing change? User can add new changes for fileset: ``` Fileset fileset = catalog.asFilesetCatalog().alterFileset( NameIdentifier.of(metaLakeName, catalogName, schemaName, filesetName), FilesetChange.removeComment()); ``` or use update fileset http request: ``` updates": [ { "@type": "removeComment" } ] ``` ### How was this patch tested? Create a fileset with comment, then use java/python client, or http requeset to remove comment and check the result --- .../gravitino/file/FilesetChange.java | 53 +++++++++++++++++++ .../hadoop/HadoopCatalogOperations.java | 2 + .../hadoop/TestHadoopCatalogOperations.java | 23 ++++++++ .../integration/test/HadoopCatalogIT.java | 29 ++++++++++ .../gravitino/client/DTOConverters.java | 2 + .../gravitino/client/TestFilesetCatalog.java | 10 ++++ .../gravitino/api/fileset_change.py | 43 +++++++++++++++ .../gravitino/catalog/fileset_catalog.py | 2 + .../dto/requests/fileset_update_request.py | 17 ++++++ .../tests/integration/test_fileset_catalog.py | 15 ++++-- .../dto/requests/FilesetUpdateRequest.java | 24 +++++++++ .../TestFilesetOperationDispatcher.java | 12 +++++ .../connector/TestCatalogOperations.java | 7 ++- ...manage-fileset-metadata-using-gravitino.md | 1 + docs/open-api/filesets.yaml | 14 +++++ .../web/rest/TestFilesetOperations.java | 19 ++++--- 16 files changed, 263 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java b/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java index 6fef3398901..258a3fe5cf8 100644 --- a/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java +++ b/api/src/main/java/com/datastrato/gravitino/file/FilesetChange.java @@ -69,6 +69,15 @@ static FilesetChange removeProperty(String property) { return new RemoveProperty(property); } + /** + * Creates a new fileset change to remove comment from the fileset. + * + * @return The fileset change. + */ + static FilesetChange removeComment() { + return RemoveComment.getInstance(); + } + /** A fileset change to rename the fileset. */ final class RenameFileset implements FilesetChange { private final String newName; @@ -300,4 +309,48 @@ public String toString() { return "REMOVEPROPERTY " + property; } } + + /** A fileset change to remove comment from the fileset. */ + final class RemoveComment implements FilesetChange { + private static final RemoveComment INSTANCE = new RemoveComment(); + + private static RemoveComment getInstance() { + return INSTANCE; + } + + /** + * Compares this RemoveComment instance with another object for equality. Two instances are + * considered equal if they are RemoveComment instance. + * + * @param o The object to compare with this instance. + * @return true if the given object represents remove comment change; false otherwise. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + return o != null && getClass() == o.getClass(); + } + + /** + * Generates a hash code for this RemoveComment instance. The hash code is based on the + * RemoveComment instance name. + * + * @return A hash code value for instance. + */ + @Override + public int hashCode() { + return Objects.hash("REMOVECOMMENT"); + } + + /** + * Provides a string representation of the RemoveComment instance. This string format includes + * the class name. + * + * @return A string summary of the comment removal operation. + */ + @Override + public String toString() { + return "REMOVECOMMENT"; + } + } } diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 4d60687a109..05683c41304 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -597,6 +597,8 @@ private FilesetEntity updateFilesetEntity( newName = ((FilesetChange.RenameFileset) change).getNewName(); } else if (change instanceof FilesetChange.UpdateFilesetComment) { newComment = ((FilesetChange.UpdateFilesetComment) change).getNewComment(); + } else if (change instanceof FilesetChange.RemoveComment) { + newComment = null; } else { throw new IllegalArgumentException( "Unsupported fileset change: " + change.getClass().getSimpleName()); diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 7023ad96ee6..24cfc72609a 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -680,6 +680,29 @@ public void testUpdateFilesetComment() throws IOException { } } + @Test + public void testRemoveFilesetComment() throws IOException { + String schemaName = "schema27"; + String comment = "comment27"; + String schemaPath = TEST_ROOT_PATH + "/" + schemaName; + createSchema(schemaName, comment, null, schemaPath); + + String name = "fileset27"; + Fileset fileset = createFileset(name, schemaName, comment, Fileset.Type.MANAGED, null, null); + + FilesetChange change1 = FilesetChange.removeComment(); + try (HadoopCatalogOperations ops = new HadoopCatalogOperations(store)) { + ops.initialize(Maps.newHashMap(), null, HADOOP_PROPERTIES_METADATA); + NameIdentifier filesetIdent = NameIdentifier.of("m1", "c1", schemaName, name); + + Fileset fileset1 = ops.alterFileset(filesetIdent, change1); + Assertions.assertEquals(name, fileset1.name()); + Assertions.assertEquals(Fileset.Type.MANAGED, fileset1.type()); + Assertions.assertNull(fileset1.comment()); + Assertions.assertEquals(fileset.storageLocation(), fileset1.storageLocation()); + } + } + private static Stream locationArguments() { return Stream.of( // Honor the catalog location diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index 310785f96b1..8b93d825350 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -529,6 +529,35 @@ public void testFilesetRemoveProperties() throws IOException { Assertions.assertEquals(0, newFileset.properties().size(), "properties should be removed"); } + @Test + public void testFilesetRemoveComment() throws IOException { + // create fileset + String filesetName = "test_remove_fileset_comment"; + String storageLocation = storageLocation(filesetName); + + createFileset( + filesetName, "comment", Fileset.Type.MANAGED, storageLocation, ImmutableMap.of("k1", "v1")); + assertFilesetExists(filesetName); + + // remove fileset comment + Fileset newFileset = + catalog + .asFilesetCatalog() + .alterFileset( + NameIdentifier.of(schemaName, filesetName), FilesetChange.removeComment()); + assertFilesetExists(filesetName); + + // verify fileset is updated + Assertions.assertNotNull(newFileset, "fileset should be created"); + Assertions.assertNull(newFileset.comment(), "comment should be removed"); + Assertions.assertEquals(Fileset.Type.MANAGED, newFileset.type(), "type should not be changed"); + Assertions.assertEquals( + storageLocation, newFileset.storageLocation(), "storage location should not be changed"); + Assertions.assertEquals(1, newFileset.properties().size(), "properties should not be changed"); + Assertions.assertEquals( + "v1", newFileset.properties().get("k1"), "properties should not be changed"); + } + @Test public void testDropCatalogWithEmptySchema() { String catalogName = diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java index 21a9f91866d..e63d254ccda 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/DTOConverters.java @@ -207,6 +207,8 @@ static FilesetUpdateRequest toFilesetUpdateRequest(FilesetChange change) { } else if (change instanceof FilesetChange.UpdateFilesetComment) { return new FilesetUpdateRequest.UpdateFilesetCommentRequest( ((FilesetChange.UpdateFilesetComment) change).getNewComment()); + } else if (change instanceof FilesetChange.RemoveComment) { + return new FilesetUpdateRequest.RemoveFilesetCommentRequest(); } else if (change instanceof FilesetChange.SetProperty) { return new FilesetUpdateRequest.SetFilesetPropertiesRequest( ((FilesetChange.SetProperty) change).getProperty(), diff --git a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java index 901253c4ceb..223c201fd37 100644 --- a/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java +++ b/clients/client-java/src/test/java/com/datastrato/gravitino/client/TestFilesetCatalog.java @@ -362,6 +362,16 @@ public void testAlterFileset() throws JsonProcessingException { Fileset res3 = catalog.asFilesetCatalog().alterFileset(fileset, req3.filesetChange()); assertFileset(mockFileset3, res3); + // Test remove fileset comment + FilesetUpdateRequest req4 = new FilesetUpdateRequest.RemoveFilesetCommentRequest(); + FilesetDTO mockFileset4 = + mockFilesetDTO("new name", Fileset.Type.MANAGED, null, "mock location", ImmutableMap.of()); + FilesetResponse resp4 = new FilesetResponse(mockFileset4); + buildMockResource( + Method.PUT, filesetPath, new FilesetUpdatesRequest(ImmutableList.of(req4)), resp4, SC_OK); + Fileset res4 = catalog.asFilesetCatalog().alterFileset(fileset, req4.filesetChange()); + assertFileset(mockFileset4, res4); + // Test NoSuchFilesetException ErrorResponse errResp = ErrorResponse.notFound(NoSuchFilesetException.class.getSimpleName(), "fileset not found"); diff --git a/clients/client-python/gravitino/api/fileset_change.py b/clients/client-python/gravitino/api/fileset_change.py index 4dba4ec9314..20e63698704 100644 --- a/clients/client-python/gravitino/api/fileset_change.py +++ b/clients/client-python/gravitino/api/fileset_change.py @@ -77,6 +77,15 @@ def remove_property(fileset_property): """ return FilesetChange.RemoveProperty(fileset_property) + @staticmethod + def remove_comment(): + """Creates a new fileset change to remove comment from the fileset. + + Returns: + The fileset change. + """ + return FilesetChange.RemoveComment() + @dataclass class RenameFileset: """A fileset change to rename the fileset.""" @@ -269,3 +278,37 @@ def __str__(self): A string summary of the property removal operation. """ return f"REMOVEPROPERTY {self._property}" + + @dataclass + class RemoveComment: + """A fileset change to remove comment from the fileset.""" + + def __eq__(self, other) -> bool: + """Compares this RemoveComment instance with another object for equality. + Two instances are considered equal if they are RemoveComment instance. + + Args: + other: The object to compare with this instance. + + Returns: + true if the given object represents the comment removal; false otherwise. + """ + return isinstance(other, FilesetChange.RemoveComment) + + def __hash__(self): + """Generates a hash code for this RemoveComment instance. + The hash code is based on the RemoveComment instance name. + + Returns: + A hash code value for comment removal operation. + """ + return hash("REMOVECOMMENT") + + def __str__(self): + """Provides a string representation of the RemoveComment instance. + This string format includes the class name. + + Returns: + A string summary of the comment removal operation. + """ + return "REMOVECOMMENT" diff --git a/clients/client-python/gravitino/catalog/fileset_catalog.py b/clients/client-python/gravitino/catalog/fileset_catalog.py index 5ab2e00e6c2..11013497c83 100644 --- a/clients/client-python/gravitino/catalog/fileset_catalog.py +++ b/clients/client-python/gravitino/catalog/fileset_catalog.py @@ -280,4 +280,6 @@ def to_fileset_update_request(change: FilesetChange): ) if isinstance(change, FilesetChange.RemoveProperty): return FilesetUpdateRequest.RemoveFilesetPropertyRequest(change.property()) + if isinstance(change, FilesetChange.RemoveComment): + return FilesetUpdateRequest.RemoveFilesetCommentRequest() raise ValueError(f"Unknown change type: {type(change).__name__}") diff --git a/clients/client-python/gravitino/dto/requests/fileset_update_request.py b/clients/client-python/gravitino/dto/requests/fileset_update_request.py index 319207bb868..f82e95b0d76 100644 --- a/clients/client-python/gravitino/dto/requests/fileset_update_request.py +++ b/clients/client-python/gravitino/dto/requests/fileset_update_request.py @@ -148,3 +148,20 @@ def validate(self): def fileset_change(self): return FilesetChange.remove_property(self._property) + + @dataclass + class RemoveFilesetCommentRequest(FilesetUpdateRequestBase): + """Represents a request to remove comment from a Fileset.""" + + def __init__(self): + super().__init__("removeComment") + + def validate(self): + """Validates the fields of the request. + + always pass + """ + pass + + def fileset_change(self): + return FilesetChange.remove_comment() diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 25641ab20c2..54a51c3559e 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -182,13 +182,15 @@ def test_failed_load_fileset(self): def test_alter_fileset(self): self.create_fileset() - fileset_propertie_new_value = self.fileset_properties_value2 + "_new" + fileset_properties_new_value = self.fileset_properties_value2 + "_new" + fileset_new_comment = self.fileset_comment + "_new" changes = ( FilesetChange.remove_property(self.fileset_properties_key1), FilesetChange.set_property( - self.fileset_properties_key2, fileset_propertie_new_value + self.fileset_properties_key2, fileset_properties_new_value ), + FilesetChange.update_comment(fileset_new_comment), ) catalog = self.gravitino_client.load_catalog(name=self.catalog_name) fileset_new = catalog.as_fileset_catalog().alter_fileset( @@ -196,6 +198,13 @@ def test_alter_fileset(self): ) self.assertEqual( fileset_new.properties().get(self.fileset_properties_key2), - fileset_propertie_new_value, + fileset_properties_new_value, ) self.assertTrue(self.fileset_properties_key1 not in fileset_new.properties()) + self.assertEqual(fileset_new.comment(), fileset_new_comment) + + fileset_comment_removed = catalog.as_fileset_catalog().alter_fileset( + self.fileset_ident, FilesetChange.remove_comment() + ) + self.assertEqual(fileset_comment_removed.name(), self.fileset_name) + self.assertIsNone(fileset_comment_removed.comment()) diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java index 38a908fba1e..948e95c6e11 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/FilesetUpdateRequest.java @@ -40,6 +40,9 @@ @JsonSubTypes.Type( value = FilesetUpdateRequest.UpdateFilesetCommentRequest.class, name = "updateComment"), + @JsonSubTypes.Type( + value = FilesetUpdateRequest.RemoveFilesetCommentRequest.class, + name = "removeComment"), @JsonSubTypes.Type( value = FilesetUpdateRequest.SetFilesetPropertiesRequest.class, name = "setProperty"), @@ -180,4 +183,25 @@ public void validate() throws IllegalArgumentException { StringUtils.isNotBlank(property), "\"property\" field is required and cannot be empty"); } } + + /** The fileset update request for removing the comment of a fileset. */ + @EqualsAndHashCode + @NoArgsConstructor(force = true) + @ToString + class RemoveFilesetCommentRequest implements FilesetUpdateRequest { + + /** @return The fileset change. */ + @Override + public FilesetChange filesetChange() { + return FilesetChange.removeComment(); + } + + /** + * Validates the request. + * + * @throws IllegalArgumentException if the request is invalid. + */ + @Override + public void validate() throws IllegalArgumentException {} + } } diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java index e69cfd27e91..6012c4d04b0 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestFilesetOperationDispatcher.java @@ -130,6 +130,18 @@ public void testCreateAndAlterFileset() { Map expectedProps = ImmutableMap.of("k2", "v2", "k3", "v3"); testProperties(expectedProps, alteredFileset.properties()); + FilesetChange[] changes2 = new FilesetChange[] {FilesetChange.updateComment("new comment")}; + + Fileset alteredFileset2 = filesetOperationDispatcher.alterFileset(filesetIdent1, changes2); + Assertions.assertEquals(fileset1.name(), alteredFileset2.name()); + Assertions.assertEquals("new comment", alteredFileset2.comment()); + + FilesetChange[] changes3 = new FilesetChange[] {FilesetChange.removeComment()}; + + Fileset alteredFileset3 = filesetOperationDispatcher.alterFileset(filesetIdent1, changes3); + Assertions.assertEquals(fileset1.name(), alteredFileset3.name()); + Assertions.assertNull(alteredFileset3.comment()); + // Test immutable fileset properties FilesetChange[] illegalChange = new FilesetChange[] {FilesetChange.setProperty(ID_KEY, "test")}; testPropertyException( diff --git a/core/src/test/java/com/datastrato/gravitino/connector/TestCatalogOperations.java b/core/src/test/java/com/datastrato/gravitino/connector/TestCatalogOperations.java index fd14c54a974..fa6e530f4a6 100644 --- a/core/src/test/java/com/datastrato/gravitino/connector/TestCatalogOperations.java +++ b/core/src/test/java/com/datastrato/gravitino/connector/TestCatalogOperations.java @@ -378,6 +378,7 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) Map newProps = fileset.properties() != null ? Maps.newHashMap(fileset.properties()) : Maps.newHashMap(); NameIdentifier newIdent = ident; + String newComment = fileset.comment(); for (FilesetChange change : changes) { if (change instanceof FilesetChange.SetProperty) { @@ -393,6 +394,10 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) throw new FilesetAlreadyExistsException("Fileset %s already exists", ident); } filesets.remove(ident); + } else if (change instanceof FilesetChange.UpdateFilesetComment) { + newComment = ((FilesetChange.UpdateFilesetComment) change).getNewComment(); + } else if (change instanceof FilesetChange.RemoveComment) { + newComment = null; } else { throw new IllegalArgumentException("Unsupported fileset change: " + change); } @@ -401,7 +406,7 @@ public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) TestFileset updatedFileset = TestFileset.builder() .withName(newIdent.name()) - .withComment(fileset.comment()) + .withComment(newComment) .withProperties(newProps) .withAuditInfo(updatedAuditInfo) .withType(fileset.type()) diff --git a/docs/manage-fileset-metadata-using-gravitino.md b/docs/manage-fileset-metadata-using-gravitino.md index 33bd670356c..a2d7bbadbd2 100644 --- a/docs/manage-fileset-metadata-using-gravitino.md +++ b/docs/manage-fileset-metadata-using-gravitino.md @@ -395,6 +395,7 @@ Currently, Gravitino supports the following changes to a fileset: | Update a comment | `{"@type":"updateComment","newComment":"new_comment"}` | `FilesetChange.updateComment("new_comment")` | | Set a fileset property | `{"@type":"setProperty","property":"key1","value":"value1"}` | `FilesetChange.setProperty("key1", "value1")` | | Remove a fileset property | `{"@type":"removeProperty","property":"key1"}` | `FilesetChange.removeProperty("key1")` | +| Remove comment | `{"@type":"removeComment"}` | `FilesetChange.removeComment()` | ### Drop a fileset diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml index 0ab70408e0f..0bf654cd84e 100644 --- a/docs/open-api/filesets.yaml +++ b/docs/open-api/filesets.yaml @@ -325,6 +325,20 @@ components: "property": "key1" } + RemoveFilesetCommentRequest: + type: object + required: + - "@type" + properties: + "@type": + type: string + description: The type of the update + enum: + - removeComment + example: { + "@type": "removeComment" + } + responses: FilesetResponse: description: The response of fileset object diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java index ec888a5916c..d9669efa168 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestFilesetOperations.java @@ -361,6 +361,14 @@ public void testUpdateFilesetComment() { assertUpdateFileset(new FilesetUpdatesRequest(ImmutableList.of(req)), fileset); } + @Test + public void testRemoveFilesetComment() { + FilesetUpdateRequest req = new FilesetUpdateRequest.RemoveFilesetCommentRequest(); + Fileset fileset = + mockFileset("fileset1", Fileset.Type.MANAGED, null, "mock location", ImmutableMap.of()); + assertUpdateFileset(new FilesetUpdatesRequest(ImmutableList.of(req)), fileset); + } + @Test public void testMultiUpdateRequest() { FilesetUpdateRequest req = new FilesetUpdateRequest.RenameFilesetRequest("new name"); @@ -371,16 +379,15 @@ public void testMultiUpdateRequest() { FilesetUpdateRequest req4 = new FilesetUpdateRequest.SetFilesetPropertiesRequest("k2", "v2"); // remove k2 FilesetUpdateRequest req5 = new FilesetUpdateRequest.RemoveFilesetPropertiesRequest("k2"); + // remove comment + FilesetUpdateRequest req6 = new FilesetUpdateRequest.RemoveFilesetCommentRequest(); Fileset fileset = mockFileset( - "new name", - Fileset.Type.MANAGED, - "new comment", - "mock location", - ImmutableMap.of("k1", "v2")); + "new name", Fileset.Type.MANAGED, null, "mock location", ImmutableMap.of("k1", "v2")); assertUpdateFileset( - new FilesetUpdatesRequest(ImmutableList.of(req, req1, req2, req3, req4, req5)), fileset); + new FilesetUpdatesRequest(ImmutableList.of(req, req1, req2, req3, req4, req5, req6)), + fileset); } @Test From 317875d885bebc5bd52f6cd520e3e590506db9e9 Mon Sep 17 00:00:00 2001 From: Dev Parikh <51128342+Dev79844@users.noreply.github.com> Date: Fri, 12 Jul 2024 13:36:24 +0530 Subject: [PATCH 14/27] [#4135] fix(trino-connector): Fix typo about Gravitino in trino-connector (#4144) ### Why are the changes needed? Fix: #4135 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing UTs --- .../gravitino/integration/test/trino/TrinoQueryTestTool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoQueryTestTool.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoQueryTestTool.java index f2311ea6a3d..89d330e1ad4 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoQueryTestTool.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoQueryTestTool.java @@ -91,7 +91,7 @@ public static void main(String[] args) throws Exception { + "TrinoTestTool --testset=tpch --tester_id=00005 --catalog=hive --auto=all\n\n" + "Run all the tpch testset's testers in the 'testsets/tpch' directory under 'mysql' " + "catalog with manual start the test environment:\n" - + "TrinoTestTool --testset=tpch -- catalog=mysql --auto=none --gravition_uri=http://10.3.21.12:8090 " + + "TrinoTestTool --testset=tpch -- catalog=mysql --auto=none --gravitino_uri=http://10.3.21.12:8090 " + "--trino_uri=http://10.3.21.12:8080 --mysql_url=jdbc:mysql:/10.3.21.12 \n"; System.out.println(example); return; From bd604fc5b090090b2a6926837e6877bcaad2d0dd Mon Sep 17 00:00:00 2001 From: roryqi Date: Fri, 12 Jul 2024 16:27:05 +0800 Subject: [PATCH 15/27] [#4105] improvement(core): Remove the logic of getValidRoles (#4121) ### What changes were proposed in this pull request? Remove the logic of getValidRoles. ### Why are the changes needed? Fix: #4105 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Modify some test cases. --- .../authorization/AccessControlManager.java | 2 +- .../authorization/PermissionManager.java | 39 ++++++++++++------ .../gravitino/authorization/RoleManager.java | 25 ------------ .../authorization/UserGroupManager.java | 40 +++---------------- ...estAccessControlManagerForPermissions.java | 36 +---------------- .../service/TestGroupMetaService.java | 6 +++ .../service/TestUserMetaService.java | 6 +++ 7 files changed, 46 insertions(+), 108 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java index eb7dbfb04ef..26ec14a7e7b 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java @@ -53,7 +53,7 @@ public class AccessControlManager { public AccessControlManager(EntityStore store, IdGenerator idGenerator, Config config) { this.adminManager = new AdminManager(store, idGenerator, config); this.roleManager = new RoleManager(store, idGenerator, config); - this.userGroupManager = new UserGroupManager(store, idGenerator, roleManager); + this.userGroupManager = new UserGroupManager(store, idGenerator); this.permissionManager = new PermissionManager(store, roleManager); } diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java index 3b24e8cde5c..95a59c18c9c 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/PermissionManager.java @@ -42,7 +42,7 @@ /** * PermissionManager is used for managing the logic the granting and revoking roles. Role is used - * for manging permissions. PermissionManager will filter the invalid roles, too. + * for manging permissions. */ class PermissionManager { private static final Logger LOG = LoggerFactory.getLogger(PermissionManager.class); @@ -67,14 +67,17 @@ User grantRolesToUser(String metalake, List roles, String user) { UserEntity.class, Entity.EntityType.USER, userEntity -> { - List roleEntities = - roleManager.getValidRoles(metalake, userEntity.roleNames(), userEntity.roleIds()); - + List roleEntities = Lists.newArrayList(); + if (userEntity.roleNames() != null) { + for (String role : userEntity.roleNames()) { + roleEntities.add(roleManager.getRole(metalake, role)); + } + } List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { - if (roleNames.contains(roleEntityToGrant.name())) { + if (roleIds.contains(roleEntityToGrant.id())) { LOG.warn( "Failed to grant, role {} already exists in the user {} of metalake {}", roleEntityToGrant.name(), @@ -129,13 +132,17 @@ Group grantRolesToGroup(String metalake, List roles, String group) { GroupEntity.class, Entity.EntityType.GROUP, groupEntity -> { - List roleEntities = - roleManager.getValidRoles(metalake, groupEntity.roleNames(), groupEntity.roleIds()); + List roleEntities = Lists.newArrayList(); + if (groupEntity.roleNames() != null) { + for (String role : groupEntity.roleNames()) { + roleEntities.add(roleManager.getRole(metalake, role)); + } + } List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); for (RoleEntity roleEntityToGrant : roleEntitiesToGrant) { - if (roleNames.contains(roleEntityToGrant.name())) { + if (roleIds.contains(roleEntityToGrant.id())) { LOG.warn( "Failed to grant, role {} already exists in the group {} of metalake {}", roleEntityToGrant.name(), @@ -190,8 +197,12 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) { GroupEntity.class, Entity.EntityType.GROUP, groupEntity -> { - List roleEntities = - roleManager.getValidRoles(metalake, groupEntity.roleNames(), groupEntity.roleIds()); + List roleEntities = Lists.newArrayList(); + if (groupEntity.roleNames() != null) { + for (String role : groupEntity.roleNames()) { + roleEntities.add(roleManager.getRole(metalake, role)); + } + } List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); @@ -252,8 +263,12 @@ User revokeRolesFromUser(String metalake, List roles, String user) { UserEntity.class, Entity.EntityType.USER, userEntity -> { - List roleEntities = - roleManager.getValidRoles(metalake, userEntity.roleNames(), userEntity.roleIds()); + List roleEntities = Lists.newArrayList(); + if (userEntity.roleNames() != null) { + for (String role : userEntity.roleNames()) { + roleEntities.add(roleManager.getRole(metalake, role)); + } + } List roleNames = Lists.newArrayList(toRoleNames(roleEntities)); List roleIds = Lists.newArrayList(toRoleIds(roleEntities)); diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java index 96fc7170250..9d717b4eb9c 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java @@ -36,7 +36,6 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Scheduler; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.time.Instant; @@ -161,28 +160,4 @@ private RoleEntity getRoleEntity(NameIdentifier identifier) { Cache getCache() { return cache; } - - List getValidRoles(String metalake, List roleNames, List roleIds) { - List roleEntities = Lists.newArrayList(); - if (roleNames == null || roleNames.isEmpty()) { - return roleEntities; - } - - int index = 0; - for (String role : roleNames) { - try { - - RoleEntity roleEntity = getRoleEntity(AuthorizationUtils.ofRole(metalake, role)); - - if (roleEntity.id().equals(roleIds.get(index))) { - roleEntities.add(roleEntity); - } - index++; - - } catch (NoSuchEntityException nse) { - // ignore this entity - } - } - return roleEntities; - } } diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java index c21b060a304..b92ab7d5a1d 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/UserGroupManager.java @@ -28,7 +28,6 @@ import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.GroupEntity; -import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.utils.PrincipalUtils; @@ -36,8 +35,6 @@ import java.io.IOException; import java.time.Instant; import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,12 +49,10 @@ class UserGroupManager { private final EntityStore store; private final IdGenerator idGenerator; - private final RoleManager roleManager; - UserGroupManager(EntityStore store, IdGenerator idGenerator, RoleManager roleManager) { + UserGroupManager(EntityStore store, IdGenerator idGenerator) { this.store = store; this.idGenerator = idGenerator; - this.roleManager = roleManager; } User addUser(String metalake, String name) throws UserAlreadyExistsException { @@ -102,20 +97,9 @@ boolean removeUser(String metalake, String user) { User getUser(String metalake, String user) throws NoSuchUserException { try { AuthorizationUtils.checkMetalakeExists(metalake); - UserEntity entity = - store.get( - AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER, UserEntity.class); + return store.get( + AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER, UserEntity.class); - List roleEntities = - roleManager.getValidRoles(metalake, entity.roles(), entity.roleIds()); - - return UserEntity.builder() - .withId(entity.id()) - .withName(entity.name()) - .withAuditInfo(entity.auditInfo()) - .withNamespace(entity.namespace()) - .withRoleNames(roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList())) - .build(); } catch (NoSuchEntityException e) { LOG.warn("User {} does not exist in the metalake {}", user, metalake, e); throw new NoSuchUserException(AuthorizationUtils.USER_DOES_NOT_EXIST_MSG, user, metalake); @@ -171,22 +155,8 @@ Group getGroup(String metalake, String group) { try { AuthorizationUtils.checkMetalakeExists(metalake); - GroupEntity entity = - store.get( - AuthorizationUtils.ofGroup(metalake, group), - Entity.EntityType.GROUP, - GroupEntity.class); - - List roleEntities = - roleManager.getValidRoles(metalake, entity.roles(), entity.roleIds()); - - return GroupEntity.builder() - .withId(entity.id()) - .withName(entity.name()) - .withAuditInfo(entity.auditInfo()) - .withNamespace(entity.namespace()) - .withRoleNames(roleEntities.stream().map(RoleEntity::name).collect(Collectors.toList())) - .build(); + return store.get( + AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP, GroupEntity.class); } catch (NoSuchEntityException e) { LOG.warn("Group {} does not exist in the metalake {}", group, metalake, e); throw new NoSuchGroupException(AuthorizationUtils.GROUP_DOES_NOT_EXIST_MSG, group, metalake); diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java index aaa46c78bda..27c27395c63 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -141,7 +141,7 @@ public void testGrantRoleToUser() { String notExist = "not-exist"; User user = accessControlManager.getUser(METALAKE, USER); - Assertions.assertTrue(user.roles().isEmpty()); + Assertions.assertNull(user.roles()); user = accessControlManager.grantRolesToUser(METALAKE, ROLE, USER); Assertions.assertFalse(user.roles().isEmpty()); @@ -275,38 +275,4 @@ public void testRevokeRoleFormGroup() { NoSuchGroupException.class, () -> accessControlManager.revokeRolesFromGroup(METALAKE, ROLE, notExist)); } - - @Test - public void testDropRole() throws IOException { - String anotherRole = "anotherRole"; - - RoleEntity roleEntity = - RoleEntity.builder() - .withNamespace( - Namespace.of( - METALAKE, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME)) - .withId(1L) - .withName(anotherRole) - .withProperties(Maps.newHashMap()) - .withSecurableObjects( - Lists.newArrayList( - SecurableObjects.ofCatalog( - CATALOG, Lists.newArrayList(Privileges.UseCatalog.allow())))) - .withAuditInfo(auditInfo) - .build(); - - entityStore.put(roleEntity, true); - - User user = - accessControlManager.grantRolesToUser(METALAKE, Lists.newArrayList(anotherRole), USER); - Assertions.assertFalse(user.roles().isEmpty()); - - Group group = - accessControlManager.grantRolesToGroup(METALAKE, Lists.newArrayList(anotherRole), GROUP); - Assertions.assertFalse(group.roles().isEmpty()); - - Assertions.assertTrue(accessControlManager.deleteRole(METALAKE, anotherRole)); - group = accessControlManager.getGroup(METALAKE, GROUP); - Assertions.assertTrue(group.roles().isEmpty()); - } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java index 3d55f8002e9..4d98103e25d 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java @@ -559,6 +559,12 @@ void updateGroup() throws IOException { Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterGroup.roleIds())); Assertions.assertEquals("creator", noUpdaterGroup.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", noUpdaterGroup.auditInfo().lastModifier()); + + // Delete a role, the group entity won't contain this role. + RoleMetaService.getInstance().deleteRole(role1.nameIdentifier()); + GroupEntity groupEntity = + GroupMetaService.getInstance().getGroupByIdentifier(group1.nameIdentifier()); + Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(groupEntity.roleNames())); } @Test diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java index 008b1eea483..fb5216801f1 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java @@ -557,6 +557,12 @@ void updateUser() throws IOException { Sets.newHashSet(role1.id(), role4.id()), Sets.newHashSet(noUpdaterUser.roleIds())); Assertions.assertEquals("creator", noUpdaterUser.auditInfo().creator()); Assertions.assertEquals("grantRevokeUser", noUpdaterUser.auditInfo().lastModifier()); + + // Delete a role, the user entity won't contain this role. + RoleMetaService.getInstance().deleteRole(role1.nameIdentifier()); + UserEntity userEntity = + UserMetaService.getInstance().getUserByIdentifier(user1.nameIdentifier()); + Assertions.assertEquals(Sets.newHashSet("role4"), Sets.newHashSet(userEntity.roleNames())); } @Test From c8d00bbecec1e3b7cefa03218745d722f0de170e Mon Sep 17 00:00:00 2001 From: JinsYin Date: Fri, 12 Jul 2024 18:05:34 +0800 Subject: [PATCH 16/27] [#4155]fix(doc): Fixed an unreachable link (#4156) ### What changes were proposed in this pull request? Fixed an unreachable link. ### Why are the changes needed? Fix: #4155 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? No testing required Co-authored-by: rqyin --- docs/lakehouse-iceberg-catalog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 15e463c9693..1434405e0dc 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -234,7 +234,7 @@ Meanwhile, the data types other than listed above are mapped to Gravitino **[Ext ### Table properties -You can pass [Iceberg table properties](https://iceberg.apache.org/docs/1.3.1/configuration/) to Gravitino when creating an Iceberg table. +You can pass [Iceberg table properties](https://web.archive.org/web/20231210013537/https://iceberg.apache.org/docs/1.3.1/configuration/) to Gravitino when creating an Iceberg table. The Gravitino server doesn't allow passing the following reserved fields. From 41431c2e613504cee74c3e4958fe96efb3386a13 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Thu, 20 Jun 2024 20:41:55 +0800 Subject: [PATCH 17/27] Add server side rest interface for tag system --- .../datastrato/gravitino/MetadataObjects.java | 42 ++ .../dto/requests/TagCreateRequest.java | 63 +++ .../dto/requests/TagUpdateRequest.java | 188 ++++++++ .../dto/requests/TagUpdatesRequest.java | 44 ++ .../dto/requests/TagsAssociateRequest.java | 64 +++ .../dto/responses/NameListResponse.java | 41 ++ .../dto/responses/TagListResponse.java | 48 ++ .../gravitino/dto/responses/TagResponse.java | 48 ++ .../gravitino/dto/tag/MetadataObjectDTO.java | 105 +++++ .../datastrato/gravitino/dto/tag/TagDTO.java | 153 +++++++ .../gravitino/dto/util/DTOConverters.java | 44 ++ .../gravitino/server/GravitinoServer.java | 4 +- .../server/web/rest/ExceptionHandlers.java | 38 ++ .../server/web/rest/OperationType.java | 3 +- .../server/web/rest/TagOperations.java | 428 ++++++++++++++++++ 15 files changed, 1311 insertions(+), 2 deletions(-) create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java create mode 100644 server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java diff --git a/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java b/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java index c71c7b1ff8f..9561e7328e0 100644 --- a/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java +++ b/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java @@ -24,6 +24,8 @@ import java.util.List; import org.apache.commons.lang3.StringUtils; +import javax.annotation.Nullable; + /** The helper class for {@link MetadataObject}. */ public class MetadataObjects { @@ -98,6 +100,46 @@ public static MetadataObject of(List names, MetadataObject.Type type) { return new MetadataObjectImpl(getParentFullName(names), getLastName(names), type); } + /** + * Get the parent metadata object of the given metadata object. + * + * @param object The metadata object + * @return The parent metadata object if it exists, otherwise null + */ + @Nullable + public static MetadataObject parent(MetadataObject object) { + if (object == null) { + return null; + } + + // Return null if the object is the root object + if (object.type() == MetadataObject.Type.METALAKE + || object.type() == MetadataObject.Type.CATALOG) { + return null; + } + + MetadataObject.Type parentType; + switch (object.type()) { + case COLUMN: + parentType = MetadataObject.Type.TABLE; + break; + case TABLE: + case FILESET: + case TOPIC: + parentType = MetadataObject.Type.SCHEMA; + break; + case SCHEMA: + parentType = MetadataObject.Type.CATALOG; + break; + + default: + throw new IllegalArgumentException( + "Unexpected to reach here for metadata object type: " + object.type()); + } + + return parse(object.parent(), parentType); + } + /** * Parse the metadata object with the given full name and type. * diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java new file mode 100644 index 00000000000..aa661b28518 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java @@ -0,0 +1,63 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.rest.RESTRequest; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +import javax.annotation.Nullable; +import java.util.Map; + +/** Represents a request to create a tag. */ +@Getter +@EqualsAndHashCode +@ToString +public class TagCreateRequest implements RESTRequest { + + @JsonProperty("name") + private final String name; + + @JsonProperty("comment") + @Nullable + private final String comment; + + @JsonProperty("properties") + @Nullable + private Map properties; + + /** + * Creates a new TagCreateRequest. + * + * @param name The name of the tag. + * @param comment The comment of the tag. + * @param properties The properties of the tag. + */ + public TagCreateRequest(String name, String comment, Map properties) { + this.name = name; + this.comment = comment; + this.properties = properties; + } + + /** This is the constructor that is used by Jackson deserializer */ + public TagCreateRequest() { + this(null, null, null); + } + + /** + * Validates the request. + * + * @throws IllegalArgumentException If the request is invalid, this exception is thrown. + */ + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(name), "\"name\" is required and cannot be empty"); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java new file mode 100644 index 00000000000..5125eebc20a --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java @@ -0,0 +1,188 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.rest.RESTRequest; +import com.datastrato.gravitino.tag.TagChange; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +/** Represents a request to update a tag. */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY) +@JsonSubTypes({ + @JsonSubTypes.Type(value = TagUpdateRequest.RenameTagRequest.class, name = "rename"), + @JsonSubTypes.Type(value = TagUpdateRequest.UpdateTagCommentRequest.class, name = "updateComment"), + @JsonSubTypes.Type(value = TagUpdateRequest.SetTagPropertyRequest.class, name = "setProperty"), + @JsonSubTypes.Type(value = TagUpdateRequest.RemoveTagPropertyRequest.class, name = "removeProperty") +}) +public interface TagUpdateRequest extends RESTRequest { + + /** + * Returns the tag change. + * + * @return the tag change. + */ + TagChange tagChange(); + + /** The tag update request for renaming a tag. */ + @EqualsAndHashCode + @ToString + class RenameTagRequest implements TagUpdateRequest { + + @Getter + @JsonProperty("newName") + private final String newName; + + /** + * Creates a new RenameTagRequest. + * + * @param newName The new name of the tag. + */ + public RenameTagRequest(String newName) { + this.newName = newName; + } + + /** This is the constructor that is used by Jackson deserializer */ + public RenameTagRequest() { + this.newName = null; + } + + @Override + public TagChange tagChange() { + return TagChange.rename(newName); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(newName), "\"newName\" must not be blank"); + } + } + + /** The tag update request for updating a tag comment. */ + @EqualsAndHashCode + @ToString + class UpdateTagCommentRequest implements TagUpdateRequest { + + @Getter + @JsonProperty("newComment") + private final String newComment; + + /** + * Creates a new UpdateTagCommentRequest. + * + * @param newComment The new comment of the tag. + */ + public UpdateTagCommentRequest(String newComment) { + this.newComment = newComment; + } + + + /** This is the constructor that is used by Jackson deserializer */ + public UpdateTagCommentRequest() { + this.newComment = null; + } + + @Override + public TagChange tagChange() { + return TagChange.updateComment(newComment); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(newComment), "\"newComment\" must not be blank"); + } + } + + /** The tag update request for setting a tag property. */ + @EqualsAndHashCode + @ToString + class SetTagPropertyRequest implements TagUpdateRequest { + + @Getter + @JsonProperty("property") + private final String property; + + @Getter + @JsonProperty("value") + private final String value; + + /** + * Creates a new SetTagPropertyRequest. + * + * @param property The property to set. + * @param value The value of the property. + */ + public SetTagPropertyRequest(String property, String value) { + this.property = property; + this.value = value; + } + + /** This is the constructor that is used by Jackson deserializer */ + public SetTagPropertyRequest() { + this.property = null; + this.value = null; + } + + @Override + public TagChange tagChange() { + return TagChange.setProperty(property, value); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(property), "\"property\" must not be blank"); + Preconditions.checkArgument( + StringUtils.isNotBlank(value), "\"value\" must not be blank"); + } + } + + /** The tag update request for removing a tag property. */ + @EqualsAndHashCode + @ToString + class RemoveTagPropertyRequest implements TagUpdateRequest { + + @Getter + @JsonProperty("property") + private final String property; + + /** + * Creates a new RemoveTagPropertyRequest. + * + * @param property The property to remove. + */ + public RemoveTagPropertyRequest(String property) { + this.property = property; + } + + /** This is the constructor that is used by Jackson deserializer */ + public RemoveTagPropertyRequest() { + this.property = null; + } + + @Override + public TagChange tagChange() { + return TagChange.removeProperty(property); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(property), "\"property\" must not be blank"); + } + } +} + + diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java new file mode 100644 index 00000000000..6f75dfcaa07 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.rest.RESTRequest; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +import java.util.List; + +/** Represents a request to update a tag. */ +@Getter +@EqualsAndHashCode +@ToString +public class TagUpdatesRequest implements RESTRequest { + + @JsonProperty("updates") + private final List updates; + + /** + * Creates a new TagUpdatesRequest. + * + * @param updates The updates to apply to the tag. + */ + public TagUpdatesRequest(List updates) { + this.updates = updates; + } + + /** This is the constructor that is used by Jackson deserializer */ + public TagUpdatesRequest() { + this(null); + } + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument(updates != null, "updates must not be null"); + updates.forEach(RESTRequest::validate); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java new file mode 100644 index 00000000000..14988303c26 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.rest.RESTRequest; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +/** Represents a request to associate tags. */ +@Getter +@EqualsAndHashCode +@ToString +public class TagsAssociateRequest implements RESTRequest { + + @JsonProperty("tagsToAdd") + private final String[] tagsToAdd; + + @JsonProperty("tagsToRemove") + private final String[] tagsToRemove; + + /** + * Creates a new TagsAssociateRequest. + * + * @param tagsToAdd The tags to add. + * @param tagsToRemove The tags to remove. + */ + public TagsAssociateRequest(String[] tagsToAdd, String[] tagsToRemove) { + this.tagsToAdd = tagsToAdd; + this.tagsToRemove = tagsToRemove; + } + + /** This is the constructor that is used by Jackson deserializer */ + public TagsAssociateRequest() { + this(null, null); + } + + /** + * Validates the request. + * + * @throws IllegalArgumentException If the request is invalid, this exception is thrown. + */ + @Override + public void validate() throws IllegalArgumentException { + if (tagsToAdd != null) { + for (String tag : tagsToAdd) { + Preconditions.checkArgument( + StringUtils.isNotBlank(tag), "tagsToAdd must not contain null or empty strings"); + } + } + + if (tagsToRemove != null) { + for (String tag : tagsToRemove) { + Preconditions.checkArgument( + StringUtils.isNotBlank(tag), "tagsToRemove must not contain null or empty strings"); + } + } + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java new file mode 100644 index 00000000000..c62b9f1b2c7 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java @@ -0,0 +1,41 @@ +package com.datastrato.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** Represents a response for a list of entity names. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class NameListResponse extends BaseResponse { + + @JsonProperty("names") + private final String[] names; + + /** + * Creates a new NameListResponse. + * + * @param names The list of names. + */ + public NameListResponse(String[] names) { + this.names = names; + } + + /** + * This is the constructor that is used by Jackson deserializer to create an instance of + * NameListResponse. + */ + public NameListResponse() { + this.names = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(names != null, "names must not be null"); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java new file mode 100644 index 00000000000..d756e48a837 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java @@ -0,0 +1,48 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.responses; + +import com.datastrato.gravitino.dto.tag.TagDTO; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** Represents a response for a list of tags. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class TagListResponse extends BaseResponse { + + @JsonProperty("tags") + private final TagDTO[] tags; + + /** + * Creates a new TagListResponse. + * + * @param tags The list of tags. + */ + public TagListResponse(TagDTO[] tags) { + super(0); + this.tags = tags; + } + + /** + * This is the constructor that is used by Jackson deserializer to create an instance of + * TagListResponse. + */ + public TagListResponse() { + super(); + this.tags = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(tags != null, "tags must not be null"); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java new file mode 100644 index 00000000000..812a2c0434e --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java @@ -0,0 +1,48 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.responses; + +import com.datastrato.gravitino.dto.tag.TagDTO; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** Represents a response for a tag. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class TagResponse extends BaseResponse { + + @JsonProperty("tag") + private final TagDTO tag; + + /** + * Creates a new TagResponse. + * + * @param tag The tag. + */ + public TagResponse(TagDTO tag) { + super(0); + this.tag = tag; + } + + /** + * This is the constructor that is used by Jackson deserializer to create an instance of + * TagResponse. + */ + public TagResponse() { + super(); + this.tag = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(tag != null, "tag must not be null"); + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java new file mode 100644 index 00000000000..52145b8b464 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java @@ -0,0 +1,105 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.dto.tag; + +import com.datastrato.gravitino.MetadataObject; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Represents a Metadata Object DTO (Data Transfer Object). */ +public class MetadataObjectDTO implements MetadataObject { + + private String parent; + + private String name; + + @JsonProperty("type") + private Type type; + + private MetadataObjectDTO() {} + + @Override + public String parent() { + return parent; + } + + @Override + public String name() { + return name; + } + + @Override + public Type type() { + return type; + } + + /** @return The full name of the metadata object. */ + @JsonProperty("fullName") + public String getFullName() { + return parent + "." + name; + } + + /** Sets the full name of the metadata object. */ + @JsonProperty("fullName") + public void setFullName(String fullName) { + int index = fullName.lastIndexOf("."); + if (index == -1) { + parent = null; + name = fullName; + } else { + parent = fullName.substring(0, index); + name = fullName.substring(index + 1); + } + } + + /** @return a new builder for constructing a Metadata Object DTO. */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for constructing a Metadata Object DTO. */ + public static class Builder { + + private final MetadataObjectDTO metadataObjectDTO = new MetadataObjectDTO(); + + /** + * Sets the parent of the metadata object. + * + * @param parent The parent of the metadata object. + * @return The builder. + */ + public Builder withParent(String parent) { + metadataObjectDTO.parent = parent; + return this; + } + + /** + * Sets the name of the metadata object. + * + * @param name The name of the metadata object. + * @return The builder. + */ + public Builder withName(String name) { + metadataObjectDTO.name = name; + return this; + } + + /** + * Sets the type of the metadata object. + * + * @param type The type of the metadata object. + * @return The builder. + */ + public Builder withType(Type type) { + metadataObjectDTO.type = type; + return this; + } + + /** @return The constructed Metadata Object DTO. */ + public MetadataObjectDTO build() { + return metadataObjectDTO; + } + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java new file mode 100644 index 00000000000..fdff8ead7d0 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java @@ -0,0 +1,153 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.dto.tag; + +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.tag.Tag; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; +import java.util.Optional; + +/** Represents a Tag Data Transfer Object (DTO). */ +public class TagDTO implements Tag, Tag.AssociatedObjects { + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @JsonProperty("audit") + private AuditDTO audit; + + @JsonProperty("inherited") + private Optional inherited; + + @JsonProperty("objects") + private MetadataObject[] objects; + + private TagDTO() {} + + @Override + public String name() { + return name; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Map properties() { + return properties; + } + + @Override + public AuditDTO auditInfo() { + return audit; + } + + @Override + public Optional inherited() { + return inherited; + } + + @Override + public MetadataObject[] objects() { + return objects; + } + + /** @return a new builder for constructing a Tag DTO. */ + public static Builder builder() { + return new Builder(); + } + + /** Builder class for constructing TagDTO instances. */ + public static class Builder { + private final TagDTO tagDTO; + + private Builder() { + tagDTO = new TagDTO(); + } + + /** + * Sets the name of the tag. + * + * @param name The name of the tag. + * @return The builder instance. + */ + public Builder withName(String name) { + tagDTO.name = name; + return this; + } + + /** + * Sets the comment associated with the tag. + * + * @param comment The comment associated with the tag. + * @return The builder instance. + */ + public Builder withComment(String comment) { + tagDTO.comment = comment; + return this; + } + + /** + * Sets the properties associated with the tag. + * + * @param properties The properties associated with the tag. + * @return The builder instance. + */ + public Builder withProperties(Map properties) { + tagDTO.properties = properties; + return this; + } + + /** + * Sets the audit information for the tag. + * + * @param audit The audit information for the tag. + * @return The builder instance. + */ + public Builder withAudit(AuditDTO audit) { + tagDTO.audit = audit; + return this; + } + + /** + * Sets whether the tag is inherited. + * + * @param inherited Whether the tag is inherited. + * @return The builder instance. + */ + public Builder withInherited(Optional inherited) { + tagDTO.inherited = inherited; + return this; + } + + /** + * Sets the objects associated with the tag. + * + * @param objects The objects associated with the tag. + * @return The builder instance. + */ + public Builder withObjects(MetadataObject[] objects) { + tagDTO.objects = objects; + return this; + } + + /** @return The constructed Tag DTO. */ + public TagDTO build() { + return tagDTO; + } + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java index 18791effc11..6b732b2d5c0 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java @@ -22,6 +22,7 @@ import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.Metalake; import com.datastrato.gravitino.Schema; import com.datastrato.gravitino.authorization.Group; @@ -65,6 +66,8 @@ import com.datastrato.gravitino.dto.rel.partitions.ListPartitionDTO; import com.datastrato.gravitino.dto.rel.partitions.PartitionDTO; import com.datastrato.gravitino.dto.rel.partitions.RangePartitionDTO; +import com.datastrato.gravitino.dto.tag.MetadataObjectDTO; +import com.datastrato.gravitino.dto.tag.TagDTO; import com.datastrato.gravitino.file.Fileset; import com.datastrato.gravitino.messaging.Topic; import com.datastrato.gravitino.rel.Column; @@ -91,6 +94,9 @@ import com.datastrato.gravitino.rel.types.Types; import java.util.Arrays; import java.util.Map; +import java.util.Optional; + +import com.datastrato.gravitino.tag.Tag; import org.apache.commons.lang3.ArrayUtils; /** Utility class for converting between DTOs and domain objects. */ @@ -465,6 +471,44 @@ public static PrivilegeDTO toDTO(Privilege privilege) { .build(); } + /** + * Converts a MetadataObject to a MetadataObjectDTO. + * + * @param metadataObject The metadata object to be converted. + * @return The metadata object DTO. + */ + public static MetadataObjectDTO toDTO(MetadataObject metadataObject) { + return MetadataObjectDTO.builder() + .withParent(metadataObject.parent()) + .withName(metadataObject.name()) + .withType(metadataObject.type()) + .build(); + } + + /** + * Converts a Tag to a TagDTO. + * + * @param tag The tag to be converted. + * @param inherited The inherited flag. + * @return The tag DTO. + */ + public static TagDTO toDTO(Tag tag, Optional inherited) { + TagDTO.Builder builder = TagDTO.builder() + .withName(tag.name()) + .withComment(tag.comment()) + .withProperties(tag.properties()) + .withAudit(toDTO(tag.auditInfo())) + .withInherited(inherited); + + Optional.ofNullable(tag.associatedObjects().objects()) + .map(Arrays::stream) + .ifPresent(objects -> + builder + .withObjects(objects.map(DTOConverters::toDTO).toArray(MetadataObjectDTO[]::new))); + + return builder.build(); + } + /** * Converts a Expression to an FunctionArg DTO. * diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index e21d71680a5..0f5260faf9e 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -44,6 +44,8 @@ import java.io.File; import java.util.Properties; import javax.servlet.Servlet; + +import com.datastrato.gravitino.tag.TagManager; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.jackson.JacksonFeature; import org.glassfish.jersey.server.ResourceConfig; @@ -97,12 +99,12 @@ private void initializeRestApi() { protected void configure() { bind(gravitinoEnv.metalakeDispatcher()).to(MetalakeDispatcher.class).ranked(1); bind(gravitinoEnv.catalogDispatcher()).to(CatalogDispatcher.class).ranked(1); - bind(gravitinoEnv.schemaDispatcher()).to(SchemaDispatcher.class).ranked(1); bind(gravitinoEnv.tableDispatcher()).to(TableDispatcher.class).ranked(1); bind(gravitinoEnv.partitionDispatcher()).to(PartitionDispatcher.class).ranked(1); bind(gravitinoEnv.filesetDispatcher()).to(FilesetDispatcher.class).ranked(1); bind(gravitinoEnv.topicDispatcher()).to(TopicDispatcher.class).ranked(1); + bind(gravitinoEnv.tagManager()).to(TagManager.class).ranked(1); } }); register(JsonProcessingExceptionMapper.class); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java index 717180df97e..69562a6475d 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java @@ -29,6 +29,7 @@ import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.exceptions.TagAlreadyExistsException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; import com.datastrato.gravitino.server.web.Utils; @@ -103,6 +104,11 @@ public static Response handleGroupPermissionOperationException( return GroupPermissionOperationExceptionHandler.INSTANCE.handle(op, roles, parent, e); } + public static Response handleTagException( + OperationType op, String tag, String parent, Exception e) { + return TagExceptionHandler.INSTANCE.handle(op, tag, parent, e); + } + private static class PartitionExceptionHandler extends BaseExceptionHandler { private static final ExceptionHandler INSTANCE = new PartitionExceptionHandler(); @@ -488,6 +494,38 @@ public Response handle(OperationType op, String roles, String parent, Exception } } + private static class TagExceptionHandler extends BaseExceptionHandler { + + private static final ExceptionHandler INSTANCE = new TagExceptionHandler(); + + private static String getTagErrorMsg( + String tag, String operation, String parent, String reason) { + return String.format( + "Failed to operate tag(s)%s operation [%s] under object [%s], reason [%s]", + tag, operation, parent, reason); + } + + @Override + public Response handle(OperationType op, String tag, String parent, Exception e) { + String formatted = StringUtil.isBlank(tag) ? "" : " [" + tag + "]"; + String errorMsg = getTagErrorMsg(formatted, op.name(), parent, getErrorMsg(e)); + LOG.warn(errorMsg, e); + + if (e instanceof IllegalArgumentException) { + return Utils.illegalArguments(errorMsg, e); + + } else if (e instanceof NotFoundException) { + return Utils.notFound(errorMsg, e); + + } else if (e instanceof TagAlreadyExistsException) { + return Utils.alreadyExists(errorMsg, e); + + } else { + return super.handle(op, tag, parent, e); + } + } + } + @VisibleForTesting static class BaseExceptionHandler extends ExceptionHandler { diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java index 1df048666fc..771ff981b62 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/OperationType.java @@ -30,5 +30,6 @@ public enum OperationType { REMOVE, DELETE, GRANT, - REVOKE + REVOKE, + ASSOCIATE, } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java new file mode 100644 index 00000000000..75339f3b2c5 --- /dev/null +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java @@ -0,0 +1,428 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.server.web.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.MetadataObjects; +import com.datastrato.gravitino.dto.requests.TagCreateRequest; +import com.datastrato.gravitino.dto.requests.TagUpdateRequest; +import com.datastrato.gravitino.dto.requests.TagUpdatesRequest; +import com.datastrato.gravitino.dto.requests.TagsAssociateRequest; +import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.NameListResponse; +import com.datastrato.gravitino.dto.responses.TagListResponse; +import com.datastrato.gravitino.dto.tag.TagDTO; +import com.datastrato.gravitino.dto.util.DTOConverters; +import com.datastrato.gravitino.exceptions.NoSuchTagException; +import com.datastrato.gravitino.metrics.MetricNames; +import com.datastrato.gravitino.server.web.Utils; +import com.datastrato.gravitino.tag.Tag; +import com.datastrato.gravitino.tag.TagChange; +import com.datastrato.gravitino.tag.TagManager; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +@Path("metalakes/{metalake}/tags") +public class TagOperations { + + private static final Logger LOG = LoggerFactory.getLogger(TagOperations.class); + + private final TagManager tagManager; + + @Context private HttpServletRequest httpRequest; + + @Inject + public TagOperations(TagManager tagManager) { + this.tagManager = tagManager; + } + + @GET + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-tags." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-tags", absolute = true) + public Response listTags( + @PathParam("metalake") String metalake, + @QueryParam("details") @DefaultValue("false") boolean verbose, + @QueryParam("extended") @DefaultValue("false") boolean extended) { + LOG.info( + "Received list tag {} with extended {} request for metalake: {}", + verbose? "infos" : "names", + extended, + metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + if (verbose) { + Tag[] tags = tagManager.listTagsInfo(metalake, extended); + TagDTO[] tagDTOs; + if (ArrayUtils.isEmpty(tags)) { + tagDTOs = new TagDTO[0]; + } else { + tagDTOs = Arrays.stream(tags) + .map(t -> DTOConverters.toDTO(t, Optional.empty())) + .toArray(TagDTO[]::new); + } + + LOG.info( + "List {} tags info with extended {} under metalake: {}", + tags.length, + extended, + metalake); + return Utils.ok(new TagListResponse(tagDTOs)); + + } else { + String[] tagNames = tagManager.listTags(metalake); + tagNames = tagNames == null ? new String[0] : tagNames; + + LOG.info("List {} tags under metalake: {}", tagNames.length, metalake); + return Utils.ok(new NameListResponse(tagNames)); + } + }); + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.LIST, "", metalake, e); + } + } + + @POST + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "create-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "create-tag", absolute = true) + public Response createTag( + @PathParam("metalake") String metalake, TagCreateRequest request) { + LOG.info("Received create tag request under metalake: {}", metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + Tag tag = tagManager.createTag( + metalake, request.getName(), request.getComment(), request.getProperties()); + + LOG.info("Created tag: {} under metalake: {}", tag.name(), metalake); + return Utils.ok(DTOConverters.toDTO(tag, Optional.empty())); + }); + } catch (Exception e) { + return ExceptionHandlers.handleTagException( + OperationType.CREATE, request.getName(), metalake, e); + } + } + + @GET + @Path("{tag}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-tag", absolute = true) + public Response getTag( + @PathParam("metalake") String metalake, + @PathParam("tag") String name, + @QueryParam("extended") @DefaultValue("false") boolean extended) { + LOG.info("Received get tag request for tag: {} under metalake: {}", name, metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + Tag tag = tagManager.getTag(metalake, name, extended); + LOG.info("Get tag: {} under metalake: {}", name, metalake); + return Utils.ok(DTOConverters.toDTO(tag, Optional.empty())); + }); + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.GET, name, metalake, e); + } + } + + @POST + @Path("{tag}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "alter-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "alter-tag", absolute = true) + public Response alterTag( + @PathParam("metalake") String metalake, + @PathParam("tag") String name, + TagUpdatesRequest request) { + LOG.info("Received alter tag request for tag: {} under metalake: {}", name, metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + + TagChange[] changes = + request.getUpdates().stream() + .map(TagUpdateRequest::tagChange) + .toArray(TagChange[]::new); + Tag tag = tagManager.alterTag(metalake, name, changes); + + LOG.info("Altered tag: {} under metalake: {}", name, metalake); + return Utils.ok(DTOConverters.toDTO(tag, Optional.empty())); + }); + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.ALTER, name, metalake, e); + } + } + + @DELETE + @Path("{tag}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-tag", absolute = true) + public Response deleteTag(@PathParam("metalake") String metalake, @PathParam("tag") String name) { + LOG.info("Received delete tag request for tag: {} under metalake: {}", name, metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = tagManager.deleteTag(metalake, name); + if (!deleted) { + LOG.warn("Failed to delete tag {} under metalake {}", name, metalake); + } else { + LOG.info("Deleted tag: {} under metalake: {}", name, metalake); + } + + return Utils.ok(new DropResponse(deleted)); + }); + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.DELETE, name, metalake, e); + } + } + + @GET + @Path("{type}/{fullName}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-object-tags." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-object-tags", absolute = true) + public Response listTagsForObject( + @PathParam("metalake") String metalake, + @PathParam("type") String type, + @PathParam("fullName") String fullName, + @QueryParam("details") @DefaultValue("false") boolean verbose) { + LOG.info( + "Received list tag {} request for object type: {}, full name: {} under metalake: {}", + verbose? "infos" : "names", + type, + fullName, + metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + MetadataObject object = MetadataObjects.parse(fullName, toType(type)); + + if (verbose) { + List tags = Lists.newArrayList(); + Tag[] nonInheritedTags = tagManager.listTagsInfoForMetadataObject(metalake, object); + if (ArrayUtils.isNotEmpty(nonInheritedTags)) { + Collections.addAll( + tags, + Arrays.stream(nonInheritedTags) + .map(t -> DTOConverters.toDTO(t, Optional.of(false))) + .toArray(TagDTO[]::new)); + } + + MetadataObject parentObject = MetadataObjects.parent(object); + while (parentObject != null) { + Tag[] heritageTags = + tagManager.listTagsInfoForMetadataObject(metalake, parentObject); + if (ArrayUtils.isNotEmpty(heritageTags)) { + Collections.addAll( + tags, + Arrays.stream(heritageTags) + .map(t -> DTOConverters.toDTO(t, Optional.of(true))) + .toArray(TagDTO[]::new)); + } + parentObject = MetadataObjects.parent(parentObject); + } + + LOG.info( + "List {} tags info for object type: {}, full name: {} under metalake: {}", + tags.size(), + type, + fullName, + metalake); + return Utils.ok(new TagListResponse(tags.toArray(new TagDTO[0]))); + + } else { + List tagNames = Lists.newArrayList(); + String[] nonInheritedTagNames = + tagManager.listTagsForMetadataObject(metalake, object); + if (ArrayUtils.isNotEmpty(nonInheritedTagNames)) { + Collections.addAll(tagNames, nonInheritedTagNames); + } + + MetadataObject parentObject = MetadataObjects.parent(object); + while (parentObject != null) { + String[] heritageTagNames = + tagManager.listTagsForMetadataObject(metalake, parentObject); + if (ArrayUtils.isNotEmpty(heritageTagNames)) { + Collections.addAll(tagNames, heritageTagNames); + } + parentObject = MetadataObjects.parent(parentObject); + } + + LOG.info( + "List {} tags for object type: {}, full name: {} under metalake: {}", + tagNames.stream(), + type, + fullName, + metalake); + return Utils.ok(new NameListResponse(tagNames.toArray(new String[0]))); + } + }); + + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.LIST, "", fullName, e); + } + } + + @GET + @Path("{type}/{fullName}/{tag}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-object-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-object-tag", absolute = true) + public Response getTagForObject( + @PathParam("metalake") String metalake, + @PathParam("type") String type, + @PathParam("fullName") String fullName, + @PathParam("tag") String tagName) { + LOG.info( + "Received get tag {} request for object type: {}, full name: {} under metalake: {}", + tagName, + type, + fullName, + metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + MetadataObject object = MetadataObjects.parse(fullName, toType(type)); + Optional tag = getTagForObject(metalake, object, tagName); + Optional tagDTO = + tag.map(t -> DTOConverters.toDTO(t, Optional.of(false))); + + MetadataObject parentObject = MetadataObjects.parent(object); + while (!tag.isPresent() && parentObject != null) { + tag = getTagForObject(metalake, parentObject, tagName); + tagDTO = tag.map(t -> DTOConverters.toDTO(t, Optional.of(true))); + parentObject = MetadataObjects.parent(parentObject); + } + + if (!tagDTO.isPresent()) { + LOG.warn("Tag {} not found for object type: {}, full name: {} under metalake: {}", + tagName, type, fullName, metalake); + return Utils.notFound( + NoSuchTagException.class.getSimpleName(), + "Tag not found: " + tagName + " for object type: " + type + + ", full name: " + fullName + " under metalake: " + metalake); + } else { + LOG.info( + "Get tag: {} for object type: {}, full name: {} under metalake: {}", + tagName, + type, + fullName, + metalake); + return Utils.ok(tagDTO.get()); + } + }); + + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.GET, tagName, fullName, e); + } + } + + @POST + @Path("{type}/{fullName}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "associate-object-tags." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "associate-object-tags", absolute = true) + public Response associateTagsForObject( + @PathParam("metalake") String metalake, + @PathParam("type") String type, + @PathParam("fullName") String fullName, + TagsAssociateRequest request) { + LOG.info( + "Received associate tags request for object type: {}, full name: {} under metalake: {}", + type, + fullName, + metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + MetadataObject object = MetadataObjects.parse(fullName, toType(type)); + String[] tagNames = tagManager.associateTagsForMetadataObject( + metalake, object, request.getTagsToAdd(), request.getTagsToRemove()); + + LOG.info( + "Associated tags: {} for object type: {}, full name: {} under metalake: {}", + Arrays.toString(tagNames), + type, + fullName, + metalake); + return Utils.ok(new NameListResponse(tagNames)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.ASSOCIATE, "", fullName, e); + } + } + + private MetadataObject.Type toType(String type) { + try { + return MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid type: " + type); + } + } + + private Optional getTagForObject(String metalake, MetadataObject object, String tagName) { + try { + return Optional.of(tagManager.getTagForMetadataObject(metalake, object, tagName)); + } catch (NoSuchTagException e) { + LOG.info("Tag {} not found for object: {}", tagName, object); + return Optional.empty(); + } + } + + + + + + + + + +} From f055105f9d8ec8122267b910cfc8f7dd916d5a1c Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Fri, 21 Jun 2024 19:47:05 +0800 Subject: [PATCH 18/27] Continue the work --- .../datastrato/gravitino/MetadataObjects.java | 3 +- .../client/ObjectMapperProvider.java | 4 +- .../dto/requests/TagCreateRequest.java | 5 +- .../dto/requests/TagUpdateRequest.java | 17 +- .../dto/requests/TagUpdatesRequest.java | 3 +- .../dto/responses/NameListResponse.java | 7 +- .../dto/responses/TagListResponse.java | 2 +- .../gravitino/dto/responses/TagResponse.java | 2 +- .../gravitino/dto/tag/MetadataObjectDTO.java | 10 +- .../datastrato/gravitino/dto/tag/TagDTO.java | 14 +- .../gravitino/dto/util/DTOConverters.java | 23 +- .../datastrato/gravitino/json/JsonUtils.java | 7 +- .../dto/requests/TestTagCreateRequest.java | 36 + .../dto/requests/TestTagUpdatesRequest.java | 78 ++ .../dto/responses/TestResponses.java | 57 + .../dto/tag/TestMetadataObjectDTO.java | 137 ++ .../gravitino/dto/tag/TestTagDTO.java | 134 ++ .../gravitino/server/GravitinoServer.java | 3 +- .../server/web/ObjectMapperProvider.java | 4 +- .../server/web/rest/TagOperations.java | 102 +- .../server/web/rest/TestTagOperations.java | 1112 +++++++++++++++++ 21 files changed, 1668 insertions(+), 92 deletions(-) create mode 100644 common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java create mode 100644 common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java create mode 100644 common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java create mode 100644 common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java create mode 100644 server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java diff --git a/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java b/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java index 9561e7328e0..2ab4643f1d4 100644 --- a/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java +++ b/api/src/main/java/com/datastrato/gravitino/MetadataObjects.java @@ -22,9 +22,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import java.util.List; -import org.apache.commons.lang3.StringUtils; - import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; /** The helper class for {@link MetadataObject}. */ public class MetadataObjects { diff --git a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java index b73c9656542..f19d8beb580 100644 --- a/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java +++ b/clients/client-java/src/main/java/com/datastrato/gravitino/client/ObjectMapperProvider.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.cfg.EnumFeature; import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; /** @@ -39,7 +40,8 @@ private static class ObjectMapperHolder { .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) .build() - .registerModule(new JavaTimeModule()); + .registerModule(new JavaTimeModule()) + .registerModule(new Jdk8Module()); } /** diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java index aa661b28518..8452162d1bb 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java @@ -7,14 +7,13 @@ import com.datastrato.gravitino.rest.RESTRequest; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import java.util.Map; +import javax.annotation.Nullable; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; import org.apache.commons.lang3.StringUtils; -import javax.annotation.Nullable; -import java.util.Map; - /** Represents a request to create a tag. */ @Getter @EqualsAndHashCode diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java index 5125eebc20a..5f058a393ff 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java @@ -21,9 +21,13 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY) @JsonSubTypes({ @JsonSubTypes.Type(value = TagUpdateRequest.RenameTagRequest.class, name = "rename"), - @JsonSubTypes.Type(value = TagUpdateRequest.UpdateTagCommentRequest.class, name = "updateComment"), + @JsonSubTypes.Type( + value = TagUpdateRequest.UpdateTagCommentRequest.class, + name = "updateComment"), @JsonSubTypes.Type(value = TagUpdateRequest.SetTagPropertyRequest.class, name = "setProperty"), - @JsonSubTypes.Type(value = TagUpdateRequest.RemoveTagPropertyRequest.class, name = "removeProperty") + @JsonSubTypes.Type( + value = TagUpdateRequest.RemoveTagPropertyRequest.class, + name = "removeProperty") }) public interface TagUpdateRequest extends RESTRequest { @@ -64,8 +68,7 @@ public TagChange tagChange() { @Override public void validate() throws IllegalArgumentException { - Preconditions.checkArgument( - StringUtils.isNotBlank(newName), "\"newName\" must not be blank"); + Preconditions.checkArgument(StringUtils.isNotBlank(newName), "\"newName\" must not be blank"); } } @@ -87,7 +90,6 @@ public UpdateTagCommentRequest(String newComment) { this.newComment = newComment; } - /** This is the constructor that is used by Jackson deserializer */ public UpdateTagCommentRequest() { this.newComment = null; @@ -144,8 +146,7 @@ public TagChange tagChange() { public void validate() throws IllegalArgumentException { Preconditions.checkArgument( StringUtils.isNotBlank(property), "\"property\" must not be blank"); - Preconditions.checkArgument( - StringUtils.isNotBlank(value), "\"value\" must not be blank"); + Preconditions.checkArgument(StringUtils.isNotBlank(value), "\"value\" must not be blank"); } } @@ -184,5 +185,3 @@ public void validate() throws IllegalArgumentException { } } } - - diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java index 6f75dfcaa07..f614ed4a754 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java @@ -7,12 +7,11 @@ import com.datastrato.gravitino.rest.RESTRequest; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import java.util.List; - /** Represents a request to update a tag. */ @Getter @EqualsAndHashCode diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java index c62b9f1b2c7..adcd733b0df 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java @@ -1,3 +1,8 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + package com.datastrato.gravitino.dto.responses; import com.fasterxml.jackson.annotation.JsonProperty; @@ -36,6 +41,6 @@ public NameListResponse() { public void validate() throws IllegalArgumentException { super.validate(); - Preconditions.checkArgument(names != null, "names must not be null"); + Preconditions.checkArgument(names != null, "\"names\" must not be null"); } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java index d756e48a837..dbc62e73d17 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java @@ -43,6 +43,6 @@ public TagListResponse() { public void validate() throws IllegalArgumentException { super.validate(); - Preconditions.checkArgument(tags != null, "tags must not be null"); + Preconditions.checkArgument(tags != null, "\"tags\" must not be null"); } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java index 812a2c0434e..fdcb68ed5c9 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java @@ -43,6 +43,6 @@ public TagResponse() { public void validate() throws IllegalArgumentException { super.validate(); - Preconditions.checkArgument(tag != null, "tag must not be null"); + Preconditions.checkArgument(tag != null, "\"tag\" must not be null"); } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java index 52145b8b464..986fd30e6b9 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java @@ -7,8 +7,10 @@ import com.datastrato.gravitino.MetadataObject; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.EqualsAndHashCode; /** Represents a Metadata Object DTO (Data Transfer Object). */ +@EqualsAndHashCode public class MetadataObjectDTO implements MetadataObject { private String parent; @@ -38,10 +40,14 @@ public Type type() { /** @return The full name of the metadata object. */ @JsonProperty("fullName") public String getFullName() { - return parent + "." + name; + return fullName(); } - /** Sets the full name of the metadata object. */ + /** + * Sets the full name of the metadata object. + * + * @param fullName The full name of the metadata object. + */ @JsonProperty("fullName") public void setFullName(String fullName) { int index = fullName.lastIndexOf("."); diff --git a/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java index fdff8ead7d0..f4e2621c6a4 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java @@ -9,11 +9,12 @@ import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.tag.Tag; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Map; import java.util.Optional; +import lombok.EqualsAndHashCode; /** Represents a Tag Data Transfer Object (DTO). */ +@EqualsAndHashCode public class TagDTO implements Tag, Tag.AssociatedObjects { @JsonProperty("name") @@ -29,10 +30,10 @@ public class TagDTO implements Tag, Tag.AssociatedObjects { private AuditDTO audit; @JsonProperty("inherited") - private Optional inherited; + private Optional inherited = Optional.empty(); @JsonProperty("objects") - private MetadataObject[] objects; + private MetadataObjectDTO[] objects; private TagDTO() {} @@ -66,6 +67,11 @@ public MetadataObject[] objects() { return objects; } + @Override + public AssociatedObjects associatedObjects() { + return this; + } + /** @return a new builder for constructing a Tag DTO. */ public static Builder builder() { return new Builder(); @@ -140,7 +146,7 @@ public Builder withInherited(Optional inherited) { * @param objects The objects associated with the tag. * @return The builder instance. */ - public Builder withObjects(MetadataObject[] objects) { + public Builder withObjects(MetadataObjectDTO[] objects) { tagDTO.objects = objects; return this; } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java index 6b732b2d5c0..c9053f5610d 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java @@ -92,11 +92,10 @@ import com.datastrato.gravitino.rel.partitions.Partitions; import com.datastrato.gravitino.rel.partitions.RangePartition; import com.datastrato.gravitino.rel.types.Types; +import com.datastrato.gravitino.tag.Tag; import java.util.Arrays; import java.util.Map; import java.util.Optional; - -import com.datastrato.gravitino.tag.Tag; import org.apache.commons.lang3.ArrayUtils; /** Utility class for converting between DTOs and domain objects. */ @@ -493,18 +492,20 @@ public static MetadataObjectDTO toDTO(MetadataObject metadataObject) { * @return The tag DTO. */ public static TagDTO toDTO(Tag tag, Optional inherited) { - TagDTO.Builder builder = TagDTO.builder() - .withName(tag.name()) - .withComment(tag.comment()) - .withProperties(tag.properties()) - .withAudit(toDTO(tag.auditInfo())) - .withInherited(inherited); + TagDTO.Builder builder = + TagDTO.builder() + .withName(tag.name()) + .withComment(tag.comment()) + .withProperties(tag.properties()) + .withAudit(toDTO(tag.auditInfo())) + .withInherited(inherited); Optional.ofNullable(tag.associatedObjects().objects()) .map(Arrays::stream) - .ifPresent(objects -> - builder - .withObjects(objects.map(DTOConverters::toDTO).toArray(MetadataObjectDTO[]::new))); + .ifPresent( + objects -> + builder.withObjects( + objects.map(DTOConverters::toDTO).toArray(MetadataObjectDTO[]::new))); return builder.build(); } diff --git a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java index 7e2e72d100a..18004a999fb 100644 --- a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java @@ -69,6 +69,7 @@ import com.fasterxml.jackson.databind.cfg.EnumFeature; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -252,7 +253,8 @@ private static class ObjectMapperHolder { .configure(EnumFeature.WRITE_ENUMS_TO_LOWERCASE, true) .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) .build() - .registerModule(new JavaTimeModule()); + .registerModule(new JavaTimeModule()) + .registerModule(new Jdk8Module()); } /** @@ -288,7 +290,8 @@ private static class AnyFieldMapperHolder { .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) .build() .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) - .registerModule(new JavaTimeModule()); + .registerModule(new JavaTimeModule()) + .registerModule(new Jdk8Module()); } /** diff --git a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java new file mode 100644 index 00000000000..5473a8fafb1 --- /dev/null +++ b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.json.JsonUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestTagCreateRequest { + + @Test + public void testTagCreateRequestSerDe() throws JsonProcessingException { + TagCreateRequest request = new TagCreateRequest("tag_test", "tag comment", null); + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + TagCreateRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, TagCreateRequest.class); + Assertions.assertEquals(request, deserRequest); + Assertions.assertEquals("tag_test", deserRequest.getName()); + Assertions.assertEquals("tag comment", deserRequest.getComment()); + Assertions.assertNull(deserRequest.getProperties()); + + Map properties = ImmutableMap.of("key", "value"); + TagCreateRequest request1 = new TagCreateRequest("tag_test", "tag comment", properties); + serJson = JsonUtils.objectMapper().writeValueAsString(request1); + TagCreateRequest deserRequest1 = + JsonUtils.objectMapper().readValue(serJson, TagCreateRequest.class); + Assertions.assertEquals(request1, deserRequest1); + Assertions.assertEquals(properties, deserRequest1.getProperties()); + } +} diff --git a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java new file mode 100644 index 00000000000..ab87acc46f3 --- /dev/null +++ b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java @@ -0,0 +1,78 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.dto.requests; + +import com.datastrato.gravitino.json.JsonUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestTagUpdatesRequest { + + @Test + public void testRenameTagRequestSerDe() throws JsonProcessingException { + TagUpdateRequest.RenameTagRequest request = + new TagUpdateRequest.RenameTagRequest("tag_test_new"); + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + TagUpdateRequest.RenameTagRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, TagUpdateRequest.RenameTagRequest.class); + Assertions.assertEquals(request, deserRequest); + Assertions.assertEquals("tag_test_new", deserRequest.getNewName()); + } + + @Test + public void testUpdateTagCommentRequestSerDe() throws JsonProcessingException { + TagUpdateRequest.UpdateTagCommentRequest request = + new TagUpdateRequest.UpdateTagCommentRequest("tag comment new"); + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + TagUpdateRequest.UpdateTagCommentRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, TagUpdateRequest.UpdateTagCommentRequest.class); + Assertions.assertEquals(request, deserRequest); + Assertions.assertEquals("tag comment new", deserRequest.getNewComment()); + } + + @Test + public void testSetTagPropertyRequestSerDe() throws JsonProcessingException { + TagUpdateRequest.SetTagPropertyRequest request = + new TagUpdateRequest.SetTagPropertyRequest("key", "value"); + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + TagUpdateRequest.SetTagPropertyRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, TagUpdateRequest.SetTagPropertyRequest.class); + Assertions.assertEquals(request, deserRequest); + Assertions.assertEquals("key", deserRequest.getProperty()); + Assertions.assertEquals("value", deserRequest.getValue()); + } + + @Test + public void testRemoveTagPropertyRequestSerDe() throws JsonProcessingException { + TagUpdateRequest.RemoveTagPropertyRequest request = + new TagUpdateRequest.RemoveTagPropertyRequest("key"); + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + TagUpdateRequest.RemoveTagPropertyRequest deserRequest = + JsonUtils.objectMapper() + .readValue(serJson, TagUpdateRequest.RemoveTagPropertyRequest.class); + Assertions.assertEquals(request, deserRequest); + Assertions.assertEquals("key", deserRequest.getProperty()); + } + + @Test + public void testTagUpdatesRequestSerDe() throws JsonProcessingException { + TagUpdateRequest request = new TagUpdateRequest.RenameTagRequest("tag_test_new"); + TagUpdateRequest request1 = new TagUpdateRequest.UpdateTagCommentRequest("tag comment new"); + TagUpdateRequest request2 = new TagUpdateRequest.SetTagPropertyRequest("key", "value"); + TagUpdateRequest request3 = new TagUpdateRequest.RemoveTagPropertyRequest("key"); + + List updates = ImmutableList.of(request, request1, request2, request3); + TagUpdatesRequest tagUpdatesRequest = new TagUpdatesRequest(updates); + String serJson = JsonUtils.objectMapper().writeValueAsString(tagUpdatesRequest); + TagUpdatesRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, TagUpdatesRequest.class); + Assertions.assertEquals(tagUpdatesRequest, deserRequest); + Assertions.assertEquals(updates, deserRequest.getUpdates()); + } +} diff --git a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java index f068213419d..4f83c7d5013 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/responses/TestResponses.java @@ -18,6 +18,8 @@ */ package com.datastrato.gravitino.dto.responses; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -39,8 +41,11 @@ import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.TableDTO; import com.datastrato.gravitino.dto.rel.partitioning.Partitioning; +import com.datastrato.gravitino.dto.tag.TagDTO; import com.datastrato.gravitino.dto.util.DTOConverters; +import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.rel.types.Types; +import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.Lists; import java.time.Instant; import org.junit.jupiter.api.Test; @@ -298,4 +303,56 @@ void testRoleResponseException() throws IllegalArgumentException { RoleResponse role = new RoleResponse(); assertThrows(IllegalArgumentException.class, () -> role.validate()); } + + @Test + void testNameListResponse() throws JsonProcessingException { + String[] names = new String[] {"name1", "name2"}; + NameListResponse response = new NameListResponse(names); + assertDoesNotThrow(response::validate); + + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + NameListResponse deserResponse = + JsonUtils.objectMapper().readValue(serJson, NameListResponse.class); + assertEquals(response, deserResponse); + assertArrayEquals(names, deserResponse.getNames()); + + NameListResponse response1 = new NameListResponse(); + Exception e = assertThrows(IllegalArgumentException.class, response1::validate); + assertEquals("\"names\" must not be null", e.getMessage()); + } + + @Test + void testTagListResponse() throws JsonProcessingException { + TagDTO tag1 = TagDTO.builder().withName("tag1").withComment("comment1").build(); + TagDTO tag2 = TagDTO.builder().withName("tag2").withComment("comment2").build(); + TagDTO[] tags = new TagDTO[] {tag1, tag2}; + TagListResponse response = new TagListResponse(tags); + assertDoesNotThrow(response::validate); + + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + TagListResponse deserResponse = + JsonUtils.objectMapper().readValue(serJson, TagListResponse.class); + assertEquals(response, deserResponse); + assertArrayEquals(tags, deserResponse.getTags()); + + TagListResponse response1 = new TagListResponse(); + Exception e = assertThrows(IllegalArgumentException.class, response1::validate); + assertEquals("\"tags\" must not be null", e.getMessage()); + } + + @Test + void testTagResponse() throws JsonProcessingException { + TagDTO tag = TagDTO.builder().withName("tag1").withComment("comment1").build(); + TagResponse response = new TagResponse(tag); + assertDoesNotThrow(response::validate); + + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + TagResponse deserResponse = JsonUtils.objectMapper().readValue(serJson, TagResponse.class); + assertEquals(response, deserResponse); + assertEquals(tag, deserResponse.getTag()); + + TagResponse response1 = new TagResponse(); + Exception e = assertThrows(IllegalArgumentException.class, response1::validate); + assertEquals("\"tag\" must not be null", e.getMessage()); + } } diff --git a/common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java new file mode 100644 index 00000000000..7c929e07f94 --- /dev/null +++ b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java @@ -0,0 +1,137 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.dto.tag; + +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.json.JsonUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestMetadataObjectDTO { + + @Test + void testObjectSerDe() throws JsonProcessingException { + + // Test metalake object + MetadataObjectDTO metalakeDTO = + MetadataObjectDTO.builder() + .withName("metalake_test") + .withType(MetadataObject.Type.METALAKE) + .build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(metalakeDTO); + String expected = "{\"fullName\":\"metalake_test\",\"type\":\"metalake\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(serJson)); + + MetadataObjectDTO deserMetalakeDTO = + JsonUtils.objectMapper().readValue(serJson, MetadataObjectDTO.class); + Assertions.assertEquals(metalakeDTO, deserMetalakeDTO); + + // Test catalog object + MetadataObjectDTO catalogDTO = + MetadataObjectDTO.builder() + .withName("catalog_test") + .withType(MetadataObject.Type.CATALOG) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(catalogDTO); + expected = "{\"fullName\":\"catalog_test\",\"type\":\"catalog\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(serJson)); + + MetadataObjectDTO deserCatalogDTO = + JsonUtils.objectMapper().readValue(serJson, MetadataObjectDTO.class); + Assertions.assertEquals(catalogDTO, deserCatalogDTO); + + // Test schema object + MetadataObjectDTO schemaDTO = + MetadataObjectDTO.builder() + .withName("schema_test") + .withParent("catalog_test") + .withType(MetadataObject.Type.SCHEMA) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(schemaDTO); + expected = "{\"fullName\":\"catalog_test.schema_test\",\"type\":\"schema\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(serJson)); + + MetadataObjectDTO deserSchemaDTO = + JsonUtils.objectMapper().readValue(serJson, MetadataObjectDTO.class); + Assertions.assertEquals(schemaDTO, deserSchemaDTO); + + // Test table object + MetadataObjectDTO tableDTO = + MetadataObjectDTO.builder() + .withName("table_test") + .withParent("catalog_test.schema_test") + .withType(MetadataObject.Type.TABLE) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(tableDTO); + expected = "{\"fullName\":\"catalog_test.schema_test.table_test\",\"type\":\"table\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(serJson)); + + MetadataObjectDTO deserTableDTO = + JsonUtils.objectMapper().readValue(serJson, MetadataObjectDTO.class); + Assertions.assertEquals(tableDTO, deserTableDTO); + + // Test column object + MetadataObjectDTO columnDTO = + MetadataObjectDTO.builder() + .withName("column_test") + .withParent("catalog_test.schema_test.table_test") + .withType(MetadataObject.Type.COLUMN) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(columnDTO); + expected = + "{\"fullName\":\"catalog_test.schema_test.table_test.column_test\",\"type\":\"column\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(serJson)); + + MetadataObjectDTO deserColumnDTO = + JsonUtils.objectMapper().readValue(serJson, MetadataObjectDTO.class); + Assertions.assertEquals(columnDTO, deserColumnDTO); + + // Test topic object + MetadataObjectDTO topicDTO = + MetadataObjectDTO.builder() + .withName("topic_test") + .withParent("catalog_test.schema_test") + .withType(MetadataObject.Type.TOPIC) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(topicDTO); + expected = "{\"fullName\":\"catalog_test.schema_test.topic_test\",\"type\":\"topic\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(serJson)); + + MetadataObjectDTO deserTopicDTO = + JsonUtils.objectMapper().readValue(serJson, MetadataObjectDTO.class); + Assertions.assertEquals(topicDTO, deserTopicDTO); + + // Test fileset object + MetadataObjectDTO filesetDTO = + MetadataObjectDTO.builder() + .withName("fileset_test") + .withParent("catalog_test.schema_test") + .withType(MetadataObject.Type.FILESET) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(filesetDTO); + expected = "{\"fullName\":\"catalog_test.schema_test.fileset_test\",\"type\":\"fileset\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(serJson)); + + MetadataObjectDTO deserFilesetDTO = + JsonUtils.objectMapper().readValue(serJson, MetadataObjectDTO.class); + Assertions.assertEquals(filesetDTO, deserFilesetDTO); + } +} diff --git a/common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java new file mode 100644 index 00000000000..d93755e1c45 --- /dev/null +++ b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java @@ -0,0 +1,134 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.dto.tag; + +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.dto.AuditDTO; +import com.datastrato.gravitino.json.JsonUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestTagDTO { + + @Test + public void testTagSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + TagDTO tagDTO = + TagDTO.builder().withName("tag_test").withComment("tag comment").withAudit(audit).build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO); + TagDTO deserTagDTO = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); + Assertions.assertEquals(tagDTO, deserTagDTO); + + Assertions.assertEquals("tag_test", deserTagDTO.name()); + Assertions.assertEquals("tag comment", deserTagDTO.comment()); + Assertions.assertEquals(audit, deserTagDTO.auditInfo()); + Assertions.assertNull(deserTagDTO.properties()); + + // Test tag with property + Map properties = ImmutableMap.of("key", "value"); + TagDTO tagDTO1 = + TagDTO.builder() + .withName("tag_test") + .withComment("tag comment") + .withAudit(audit) + .withProperties(properties) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO1); + TagDTO deserTagDTO1 = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); + Assertions.assertEquals(tagDTO1, deserTagDTO1); + + Assertions.assertEquals(properties, deserTagDTO1.properties()); + + // Test tag with inherited + TagDTO tagDTO2 = + TagDTO.builder() + .withName("tag_test") + .withComment("tag comment") + .withAudit(audit) + .withInherited(Optional.empty()) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO2); + TagDTO deserTagDTO2 = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); + Assertions.assertEquals(tagDTO2, deserTagDTO2); + Assertions.assertEquals(Optional.empty(), deserTagDTO2.inherited()); + + TagDTO tagDTO3 = + TagDTO.builder() + .withName("tag_test") + .withComment("tag comment") + .withAudit(audit) + .withInherited(Optional.of(false)) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO3); + TagDTO deserTagDTO3 = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); + Assertions.assertEquals(Optional.of(false), deserTagDTO3.inherited()); + + TagDTO tagDTO4 = + TagDTO.builder() + .withName("tag_test") + .withComment("tag comment") + .withAudit(audit) + .withInherited(Optional.of(true)) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO4); + TagDTO deserTagDTO4 = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); + Assertions.assertEquals(Optional.of(true), deserTagDTO4.inherited()); + + // Test tag with metadata objects + MetadataObjectDTO metalakeDTO = + MetadataObjectDTO.builder() + .withName("metalake_test") + .withType(MetadataObject.Type.METALAKE) + .build(); + + MetadataObjectDTO catalogDTO = + MetadataObjectDTO.builder() + .withName("catalog_test") + .withType(MetadataObject.Type.CATALOG) + .build(); + + MetadataObjectDTO schemaDTO = + MetadataObjectDTO.builder() + .withName("schema_test") + .withParent("catalog_test") + .withType(MetadataObject.Type.SCHEMA) + .build(); + + MetadataObjectDTO tableDTO = + MetadataObjectDTO.builder() + .withName("table_test") + .withParent("catalog_test.schema_test") + .withType(MetadataObject.Type.TABLE) + .build(); + + MetadataObjectDTO[] objectDTOs = + new MetadataObjectDTO[] {metalakeDTO, catalogDTO, schemaDTO, tableDTO}; + + TagDTO tagDTO5 = + TagDTO.builder() + .withName("tag_test") + .withComment("tag comment") + .withAudit(audit) + .withObjects(objectDTOs) + .build(); + + serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO5); + TagDTO deserTagDTO5 = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); + Assertions.assertEquals(tagDTO5, deserTagDTO5); + Assertions.assertArrayEquals(objectDTOs, deserTagDTO5.objects()); + } +} diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 0f5260faf9e..ab935ec8dfe 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -41,11 +41,10 @@ import com.datastrato.gravitino.server.web.mapper.JsonParseExceptionMapper; import com.datastrato.gravitino.server.web.mapper.JsonProcessingExceptionMapper; import com.datastrato.gravitino.server.web.ui.WebUIFilter; +import com.datastrato.gravitino.tag.TagManager; import java.io.File; import java.util.Properties; import javax.servlet.Servlet; - -import com.datastrato.gravitino.tag.TagManager; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.jackson.JacksonFeature; import org.glassfish.jersey.server.ResourceConfig; diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/ObjectMapperProvider.java b/server/src/main/java/com/datastrato/gravitino/server/web/ObjectMapperProvider.java index ac07a9d32d9..632f023e28b 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/ObjectMapperProvider.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/ObjectMapperProvider.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.cfg.EnumFeature; import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; @@ -39,7 +40,8 @@ private static class ObjectMapperHolder { .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) .build() .setSerializationInclusion(JsonInclude.Include.NON_NULL) - .registerModule(new JavaTimeModule()); + .registerModule(new JavaTimeModule()) + .registerModule(new Jdk8Module()); } /** diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java index 75339f3b2c5..9c0a460944c 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java @@ -15,6 +15,7 @@ import com.datastrato.gravitino.dto.responses.DropResponse; import com.datastrato.gravitino.dto.responses.NameListResponse; import com.datastrato.gravitino.dto.responses.TagListResponse; +import com.datastrato.gravitino.dto.responses.TagResponse; import com.datastrato.gravitino.dto.tag.TagDTO; import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.exceptions.NoSuchTagException; @@ -24,13 +25,13 @@ import com.datastrato.gravitino.tag.TagChange; import com.datastrato.gravitino.tag.TagManager; import com.google.common.collect.Lists; -import org.apache.commons.lang3.ArrayUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -41,11 +42,9 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Optional; +import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Path("metalakes/{metalake}/tags") public class TagOperations { @@ -71,7 +70,7 @@ public Response listTags( @QueryParam("extended") @DefaultValue("false") boolean extended) { LOG.info( "Received list tag {} with extended {} request for metalake: {}", - verbose? "infos" : "names", + verbose ? "infos" : "names", extended, metalake); @@ -85,14 +84,15 @@ public Response listTags( if (ArrayUtils.isEmpty(tags)) { tagDTOs = new TagDTO[0]; } else { - tagDTOs = Arrays.stream(tags) - .map(t -> DTOConverters.toDTO(t, Optional.empty())) - .toArray(TagDTO[]::new); + tagDTOs = + Arrays.stream(tags) + .map(t -> DTOConverters.toDTO(t, Optional.empty())) + .toArray(TagDTO[]::new); } LOG.info( "List {} tags info with extended {} under metalake: {}", - tags.length, + tagDTOs.length, extended, metalake); return Utils.ok(new TagListResponse(tagDTOs)); @@ -114,8 +114,7 @@ public Response listTags( @Produces("application/vnd.gravitino.v1+json") @Timed(name = "create-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "create-tag", absolute = true) - public Response createTag( - @PathParam("metalake") String metalake, TagCreateRequest request) { + public Response createTag(@PathParam("metalake") String metalake, TagCreateRequest request) { LOG.info("Received create tag request under metalake: {}", metalake); try { @@ -123,11 +122,12 @@ public Response createTag( httpRequest, () -> { request.validate(); - Tag tag = tagManager.createTag( - metalake, request.getName(), request.getComment(), request.getProperties()); + Tag tag = + tagManager.createTag( + metalake, request.getName(), request.getComment(), request.getProperties()); LOG.info("Created tag: {} under metalake: {}", tag.name(), metalake); - return Utils.ok(DTOConverters.toDTO(tag, Optional.empty())); + return Utils.ok(new TagResponse(DTOConverters.toDTO(tag, Optional.empty()))); }); } catch (Exception e) { return ExceptionHandlers.handleTagException( @@ -152,7 +152,7 @@ public Response getTag( () -> { Tag tag = tagManager.getTag(metalake, name, extended); LOG.info("Get tag: {} under metalake: {}", name, metalake); - return Utils.ok(DTOConverters.toDTO(tag, Optional.empty())); + return Utils.ok(new TagResponse(DTOConverters.toDTO(tag, Optional.empty()))); }); } catch (Exception e) { return ExceptionHandlers.handleTagException(OperationType.GET, name, metalake, e); @@ -178,12 +178,12 @@ public Response alterTag( TagChange[] changes = request.getUpdates().stream() - .map(TagUpdateRequest::tagChange) - .toArray(TagChange[]::new); + .map(TagUpdateRequest::tagChange) + .toArray(TagChange[]::new); Tag tag = tagManager.alterTag(metalake, name, changes); LOG.info("Altered tag: {} under metalake: {}", name, metalake); - return Utils.ok(DTOConverters.toDTO(tag, Optional.empty())); + return Utils.ok(new TagResponse(DTOConverters.toDTO(tag, Optional.empty()))); }); } catch (Exception e) { return ExceptionHandlers.handleTagException(OperationType.ALTER, name, metalake, e); @@ -228,7 +228,7 @@ public Response listTagsForObject( @QueryParam("details") @DefaultValue("false") boolean verbose) { LOG.info( "Received list tag {} request for object type: {}, full name: {} under metalake: {}", - verbose? "infos" : "names", + verbose ? "infos" : "names", type, fullName, metalake); @@ -252,12 +252,12 @@ public Response listTagsForObject( MetadataObject parentObject = MetadataObjects.parent(object); while (parentObject != null) { - Tag[] heritageTags = + Tag[] inheritedTags = tagManager.listTagsInfoForMetadataObject(metalake, parentObject); - if (ArrayUtils.isNotEmpty(heritageTags)) { + if (ArrayUtils.isNotEmpty(inheritedTags)) { Collections.addAll( tags, - Arrays.stream(heritageTags) + Arrays.stream(inheritedTags) .map(t -> DTOConverters.toDTO(t, Optional.of(true))) .toArray(TagDTO[]::new)); } @@ -282,10 +282,10 @@ public Response listTagsForObject( MetadataObject parentObject = MetadataObjects.parent(object); while (parentObject != null) { - String[] heritageTagNames = + String[] inheritedTagNames = tagManager.listTagsForMetadataObject(metalake, parentObject); - if (ArrayUtils.isNotEmpty(heritageTagNames)) { - Collections.addAll(tagNames, heritageTagNames); + if (ArrayUtils.isNotEmpty(inheritedTagNames)) { + Collections.addAll(tagNames, inheritedTagNames); } parentObject = MetadataObjects.parent(parentObject); } @@ -328,8 +328,7 @@ public Response getTagForObject( () -> { MetadataObject object = MetadataObjects.parse(fullName, toType(type)); Optional tag = getTagForObject(metalake, object, tagName); - Optional tagDTO = - tag.map(t -> DTOConverters.toDTO(t, Optional.of(false))); + Optional tagDTO = tag.map(t -> DTOConverters.toDTO(t, Optional.of(false))); MetadataObject parentObject = MetadataObjects.parent(object); while (!tag.isPresent() && parentObject != null) { @@ -339,12 +338,22 @@ public Response getTagForObject( } if (!tagDTO.isPresent()) { - LOG.warn("Tag {} not found for object type: {}, full name: {} under metalake: {}", - tagName, type, fullName, metalake); + LOG.warn( + "Tag {} not found for object type: {}, full name: {} under metalake: {}", + tagName, + type, + fullName, + metalake); return Utils.notFound( NoSuchTagException.class.getSimpleName(), - "Tag not found: " + tagName + " for object type: " + type + - ", full name: " + fullName + " under metalake: " + metalake); + "Tag not found: " + + tagName + + " for object type: " + + type + + ", full name: " + + fullName + + " under metalake: " + + metalake); } else { LOG.info( "Get tag: {} for object type: {}, full name: {} under metalake: {}", @@ -352,7 +361,7 @@ public Response getTagForObject( type, fullName, metalake); - return Utils.ok(tagDTO.get()); + return Utils.ok(new TagResponse(tagDTO.get())); } }); @@ -383,8 +392,10 @@ public Response associateTagsForObject( () -> { request.validate(); MetadataObject object = MetadataObjects.parse(fullName, toType(type)); - String[] tagNames = tagManager.associateTagsForMetadataObject( - metalake, object, request.getTagsToAdd(), request.getTagsToRemove()); + String[] tagNames = + tagManager.associateTagsForMetadataObject( + metalake, object, request.getTagsToAdd(), request.getTagsToRemove()); + tagNames = tagNames == null ? new String[0] : tagNames; LOG.info( "Associated tags: {} for object type: {}, full name: {} under metalake: {}", @@ -410,19 +421,10 @@ private MetadataObject.Type toType(String type) { private Optional getTagForObject(String metalake, MetadataObject object, String tagName) { try { - return Optional.of(tagManager.getTagForMetadataObject(metalake, object, tagName)); + return Optional.ofNullable(tagManager.getTagForMetadataObject(metalake, object, tagName)); } catch (NoSuchTagException e) { LOG.info("Tag {} not found for object: {}", tagName, object); return Optional.empty(); } } - - - - - - - - - } diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java new file mode 100644 index 00000000000..7da6e8f2f0e --- /dev/null +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java @@ -0,0 +1,1112 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.server.web.rest; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.MetadataObjects; +import com.datastrato.gravitino.dto.requests.TagCreateRequest; +import com.datastrato.gravitino.dto.requests.TagUpdateRequest; +import com.datastrato.gravitino.dto.requests.TagUpdatesRequest; +import com.datastrato.gravitino.dto.requests.TagsAssociateRequest; +import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.ErrorConstants; +import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.NameListResponse; +import com.datastrato.gravitino.dto.responses.TagListResponse; +import com.datastrato.gravitino.dto.responses.TagResponse; +import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; +import com.datastrato.gravitino.exceptions.NoSuchTagException; +import com.datastrato.gravitino.exceptions.TagAlreadyExistsException; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.TagEntity; +import com.datastrato.gravitino.rest.RESTUtils; +import com.datastrato.gravitino.tag.Tag; +import com.datastrato.gravitino.tag.TagChange; +import com.datastrato.gravitino.tag.TagManager; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.glassfish.jersey.internal.inject.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.glassfish.jersey.test.TestProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestTagOperations extends JerseyTest { + + private static class MockServletRequestFactory extends ServletRequestFactoryBase { + + @Override + public HttpServletRequest get() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn(null); + return request; + } + } + + private TagManager tagManager = mock(TagManager.class); + + private String metalake = "test_metalake"; + + private AuditInfo testAuditInfo1 = + AuditInfo.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + @Override + protected Application configure() { + try { + forceSet( + TestProperties.CONTAINER_PORT, String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(TagOperations.class); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bind(tagManager).to(TagManager.class).ranked(2); + bindFactory(TestTagOperations.MockServletRequestFactory.class) + .to(HttpServletRequest.class); + } + }); + + return resourceConfig; + } + + @Test + public void testListTags() { + String[] tags = new String[] {"tag1", "tag2"}; + when(tagManager.listTags(metalake)).thenReturn(tags); + + Response response = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + NameListResponse nameListResponse = response.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse.getCode()); + Assertions.assertArrayEquals(tags, nameListResponse.getNames()); + + when(tagManager.listTags(metalake)).thenReturn(null); + Response resp1 = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + + NameListResponse nameListResponse1 = resp1.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse1.getCode()); + Assertions.assertEquals(0, nameListResponse1.getNames().length); + + when(tagManager.listTags(metalake)).thenReturn(new String[0]); + Response resp2 = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + + NameListResponse nameListResponse2 = resp2.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse2.getCode()); + Assertions.assertEquals(0, nameListResponse2.getNames().length); + + // Test throw NoSuchMetalakeException + doThrow(new NoSuchMetalakeException("mock error")).when(tagManager).listTags(metalake); + Response resp3 = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchMetalakeException.class.getSimpleName(), errorResp.getType()); + + // Test throw RuntimeException + doThrow(new RuntimeException("mock error")).when(tagManager).listTags(metalake); + Response resp4 = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp1 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testListTagsInfo() { + TagEntity tag1 = + TagEntity.builder() + .withName("tag1") + .withId(1L) + .withComment("tag1 comment") + .withAuditInfo(testAuditInfo1) + .build(); + + TagEntity tag2 = + TagEntity.builder() + .withName("tag2") + .withId(1L) + .withComment("tag2 comment") + .withAuditInfo(testAuditInfo1) + .build(); + + Tag[] tags = new Tag[] {tag1, tag2}; + when(tagManager.listTagsInfo(metalake, false)).thenReturn(tags); + + Response resp = + target(tagPath(metalake)) + .queryParam("extended", false) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + TagListResponse tagListResp = resp.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResp.getCode()); + Assertions.assertEquals(tags.length, tagListResp.getTags().length); + + Assertions.assertEquals(tag1.name(), tagListResp.getTags()[0].name()); + Assertions.assertEquals(tag1.comment(), tagListResp.getTags()[0].comment()); + Assertions.assertEquals(Optional.empty(), tagListResp.getTags()[0].inherited()); + + Assertions.assertEquals(tag2.name(), tagListResp.getTags()[1].name()); + Assertions.assertEquals(tag2.comment(), tagListResp.getTags()[1].comment()); + Assertions.assertEquals(Optional.empty(), tagListResp.getTags()[1].inherited()); + + // Test return null + when(tagManager.listTagsInfo(metalake, false)).thenReturn(null); + Response resp1 = + target(tagPath(metalake)) + .queryParam("extended", false) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + + TagListResponse tagListResp1 = resp1.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResp1.getCode()); + Assertions.assertEquals(0, tagListResp1.getTags().length); + + // Test return empty array + when(tagManager.listTagsInfo(metalake, false)).thenReturn(new Tag[0]); + Response resp2 = + target(tagPath(metalake)) + .queryParam("extended", false) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + + TagListResponse tagListResp2 = resp2.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResp2.getCode()); + Assertions.assertEquals(0, tagListResp2.getTags().length); + + // Test with extended = true + MetadataObject[] objects = + new MetadataObject[] {MetadataObjects.parse("catalog_1", MetadataObject.Type.CATALOG)}; + + TagEntity tag3 = + TagEntity.builder() + .withName("tag3") + .withId(1L) + .withComment("tag3 comment") + .withAuditInfo(testAuditInfo1) + .withMetadataObjects(objects) + .build(); + + TagEntity tag4 = + TagEntity.builder() + .withName("tag4") + .withId(1L) + .withComment("tag4 comment") + .withAuditInfo(testAuditInfo1) + .withMetadataObjects(objects) + .build(); + + Tag[] tags1 = new Tag[] {tag3, tag4}; + when(tagManager.listTagsInfo(metalake, true)).thenReturn(tags1); + + Response resp3 = + target(tagPath(metalake)) + .queryParam("extended", true) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + + TagListResponse tagListResp3 = resp3.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResp3.getCode()); + Assertions.assertEquals(tags1.length, tagListResp3.getTags().length); + + Assertions.assertEquals(tag3.name(), tagListResp3.getTags()[0].name()); + Assertions.assertEquals(tag3.comment(), tagListResp3.getTags()[0].comment()); + Assertions.assertEquals(tag3.objects().length, tagListResp3.getTags()[0].objects().length); + Assertions.assertEquals(Optional.empty(), tagListResp3.getTags()[0].inherited()); + + Assertions.assertEquals(tag4.name(), tagListResp3.getTags()[1].name()); + Assertions.assertEquals(tag4.comment(), tagListResp3.getTags()[1].comment()); + Assertions.assertEquals(tag4.objects().length, tagListResp3.getTags()[1].objects().length); + Assertions.assertEquals(Optional.empty(), tagListResp3.getTags()[1].inherited()); + } + + @Test + public void testCreateTag() { + TagEntity tag1 = + TagEntity.builder() + .withName("tag1") + .withId(1L) + .withComment("tag1 comment") + .withAuditInfo(testAuditInfo1) + .build(); + when(tagManager.createTag(metalake, "tag1", "tag1 comment", null)).thenReturn(tag1); + + TagCreateRequest request = new TagCreateRequest("tag1", "tag1 comment", null); + Response resp = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + TagResponse tagResp = resp.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResp.getCode()); + + Tag respTag = tagResp.getTag(); + Assertions.assertEquals(tag1.name(), respTag.name()); + Assertions.assertEquals(tag1.comment(), respTag.comment()); + Assertions.assertEquals(Optional.empty(), respTag.inherited()); + Assertions.assertNull(respTag.associatedObjects().objects()); + Assertions.assertEquals(0, respTag.associatedObjects().count()); + + // Test throw TagAlreadyExistsException + doThrow(new TagAlreadyExistsException("mock error")) + .when(tagManager) + .createTag(any(), any(), any(), any()); + Response resp1 = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp.getCode()); + Assertions.assertEquals(TagAlreadyExistsException.class.getSimpleName(), errorResp.getType()); + + // Test throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(tagManager) + .createTag(any(), any(), any(), any()); + + Response resp2 = + target(tagPath(metalake)) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetTag() { + TagEntity tag1 = + TagEntity.builder() + .withName("tag1") + .withId(1L) + .withComment("tag1 comment") + .withAuditInfo(testAuditInfo1) + .build(); + when(tagManager.getTag(metalake, "tag1", false)).thenReturn(tag1); + + Response resp = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + TagResponse tagResp = resp.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResp.getCode()); + + Tag respTag = tagResp.getTag(); + Assertions.assertEquals(tag1.name(), respTag.name()); + Assertions.assertEquals(tag1.comment(), respTag.comment()); + Assertions.assertEquals(Optional.empty(), respTag.inherited()); + + // Test get tag with extended = true + MetadataObject[] objects = + new MetadataObject[] {MetadataObjects.parse("catalog_1", MetadataObject.Type.CATALOG)}; + + TagEntity tag2 = + TagEntity.builder() + .withName("tag2") + .withId(1L) + .withComment("tag2 comment") + .withAuditInfo(testAuditInfo1) + .withMetadataObjects(objects) + .build(); + when(tagManager.getTag(metalake, "tag2", true)).thenReturn(tag2); + + Response resp1 = + target(tagPath(metalake)) + .path("tag2") + .queryParam("extended", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + + TagResponse tagResp1 = resp1.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResp1.getCode()); + + Tag respTag1 = tagResp1.getTag(); + Assertions.assertEquals(tag2.name(), respTag1.name()); + Assertions.assertEquals(tag2.comment(), respTag1.comment()); + Assertions.assertEquals(tag2.objects().length, respTag1.associatedObjects().objects().length); + Assertions.assertEquals(Optional.empty(), respTag1.inherited()); + + // Test throw NoSuchTagException + doThrow(new NoSuchTagException("mock error")).when(tagManager).getTag(metalake, "tag1", false); + + Response resp2 = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchTagException.class.getSimpleName(), errorResp.getType()); + + // Test throw RuntimeException + doThrow(new RuntimeException("mock error")).when(tagManager).getTag(metalake, "tag1", false); + + Response resp3 = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp1 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testAlterTag() { + TagEntity newTag = + TagEntity.builder() + .withName("new_tag1") + .withId(1L) + .withComment("new tag1 comment") + .withAuditInfo(testAuditInfo1) + .build(); + + TagChange[] changes = + new TagChange[] {TagChange.rename("new_tag1"), TagChange.updateComment("new tag1 comment")}; + + when(tagManager.alterTag(metalake, "tag1", changes)).thenReturn(newTag); + + TagUpdateRequest[] requests = + new TagUpdateRequest[] { + new TagUpdateRequest.RenameTagRequest("new_tag1"), + new TagUpdateRequest.UpdateTagCommentRequest("new tag1 comment") + }; + TagUpdatesRequest request = new TagUpdatesRequest(Lists.newArrayList(requests)); + Response resp = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + TagResponse tagResp = resp.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResp.getCode()); + + Tag respTag = tagResp.getTag(); + Assertions.assertEquals(newTag.name(), respTag.name()); + Assertions.assertEquals(newTag.comment(), respTag.comment()); + Assertions.assertEquals(Optional.empty(), respTag.inherited()); + + // Test throw NoSuchTagException + doThrow(new NoSuchTagException("mock error")).when(tagManager).alterTag(any(), any(), any()); + + Response resp1 = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchTagException.class.getSimpleName(), errorResp.getType()); + + // Test throw RuntimeException + doThrow(new RuntimeException("mock error")).when(tagManager).alterTag(any(), any(), any()); + + Response resp2 = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testDeleteTag() { + when(tagManager.deleteTag(metalake, "tag1")).thenReturn(true); + + Response resp = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + when(tagManager.deleteTag(metalake, "tag1")).thenReturn(false); + Response resp1 = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test throw RuntimeException + doThrow(new RuntimeException("mock error")).when(tagManager).deleteTag(any(), any()); + + Response resp2 = + target(tagPath(metalake)) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testListTagsForObject() { + String[] catalogTags = new String[] {"tag1", "tag2"}; + MetadataObject catalog = MetadataObjects.parse("object1", MetadataObject.Type.CATALOG); + when(tagManager.listTagsForMetadataObject(metalake, catalog)).thenReturn(catalogTags); + + String[] schemaTags = new String[] {"tag3", "tag4"}; + MetadataObject schema = MetadataObjects.parse("object1.object2", MetadataObject.Type.SCHEMA); + when(tagManager.listTagsForMetadataObject(metalake, schema)).thenReturn(schemaTags); + + String[] tableTags = new String[] {"tag5", "tag6"}; + MetadataObject table = + MetadataObjects.parse("object1.object2.object3", MetadataObject.Type.TABLE); + when(tagManager.listTagsForMetadataObject(metalake, table)).thenReturn(tableTags); + + String[] columnTags = new String[] {"tag7", "tag8"}; + MetadataObject column = + MetadataObjects.parse("object1.object2.object3.object4", MetadataObject.Type.COLUMN); + when(tagManager.listTagsForMetadataObject(metalake, column)).thenReturn(columnTags); + + // Test catalog tags + Response response = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + NameListResponse nameListResponse = response.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse.getCode()); + Assertions.assertArrayEquals(catalogTags, nameListResponse.getNames()); + + // Test schema tags + Response response1 = + target(tagPath(metalake)) + .path(schema.type().toString()) + .path(schema.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response1.getStatus()); + + NameListResponse nameListResponse1 = response1.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse1.getCode()); + + Set expected = Sets.newHashSet(catalogTags); + expected.addAll(Sets.newHashSet(schemaTags)); + Assertions.assertEquals(expected, Sets.newHashSet(nameListResponse1.getNames())); + + // Test table tags + Response response2 = + target(tagPath(metalake)) + .path(table.type().toString()) + .path(table.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response2.getStatus()); + + NameListResponse nameListResponse2 = response2.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse2.getCode()); + + Set expected1 = Sets.newHashSet(catalogTags); + expected1.addAll(Sets.newHashSet(schemaTags)); + expected1.addAll(Sets.newHashSet(tableTags)); + Assertions.assertEquals(expected1, Sets.newHashSet(nameListResponse2.getNames())); + + // Test column tags + Response response3 = + target(tagPath(metalake)) + .path(column.type().toString()) + .path(column.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response3.getStatus()); + + NameListResponse nameListResponse3 = response3.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse3.getCode()); + + Set expected2 = Sets.newHashSet(catalogTags); + expected2.addAll(Sets.newHashSet(schemaTags)); + expected2.addAll(Sets.newHashSet(tableTags)); + expected2.addAll(Sets.newHashSet(columnTags)); + Assertions.assertEquals(expected2, Sets.newHashSet(nameListResponse3.getNames())); + + // Test with null tags + when(tagManager.listTagsForMetadataObject(metalake, catalog)).thenReturn(null); + Response response4 = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response4.getStatus()); + + NameListResponse nameListResponse4 = response4.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse4.getCode()); + + Assertions.assertEquals(0, nameListResponse4.getNames().length); + + Response response5 = + target(tagPath(metalake)) + .path(schema.type().toString()) + .path(schema.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response5.getStatus()); + + NameListResponse nameListResponse5 = response5.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse5.getCode()); + + Set expected3 = Sets.newHashSet(schemaTags); + Assertions.assertEquals(expected3, Sets.newHashSet(nameListResponse5.getNames())); + + // Test with "details" = true + Tag[] catalogTagInfos = + new Tag[] { + TagEntity.builder().withName("tag1").withId(1L).withAuditInfo(testAuditInfo1).build() + }; + when(tagManager.listTagsInfoForMetadataObject(metalake, catalog)).thenReturn(catalogTagInfos); + + Tag[] schemaTagInfos = + new Tag[] { + TagEntity.builder().withName("tag3").withId(1L).withAuditInfo(testAuditInfo1).build() + }; + when(tagManager.listTagsInfoForMetadataObject(metalake, schema)).thenReturn(schemaTagInfos); + + Tag[] tableTagInfos = + new Tag[] { + TagEntity.builder().withName("tag5").withId(1L).withAuditInfo(testAuditInfo1).build() + }; + when(tagManager.listTagsInfoForMetadataObject(metalake, table)).thenReturn(tableTagInfos); + + Tag[] columnTagInfos = + new Tag[] { + TagEntity.builder().withName("tag7").withId(1L).withAuditInfo(testAuditInfo1).build() + }; + when(tagManager.listTagsInfoForMetadataObject(metalake, column)).thenReturn(columnTagInfos); + + // Test catalog tags + Response response6 = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response6.getStatus()); + + TagListResponse tagListResponse = response6.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResponse.getCode()); + Assertions.assertEquals(catalogTagInfos.length, tagListResponse.getTags().length); + + Map resultTags = + Arrays.stream(tagListResponse.getTags()) + .collect(Collectors.toMap(Tag::name, Function.identity())); + + Assertions.assertTrue(resultTags.containsKey("tag1")); + Assertions.assertFalse(resultTags.get("tag1").inherited().get()); + + // Test schema tags + Response response7 = + target(tagPath(metalake)) + .path(schema.type().toString()) + .path(schema.fullName()) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response7.getStatus()); + + TagListResponse tagListResponse1 = response7.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResponse1.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length, tagListResponse1.getTags().length); + + Map resultTags1 = + Arrays.stream(tagListResponse1.getTags()) + .collect(Collectors.toMap(Tag::name, Function.identity())); + + Assertions.assertTrue(resultTags1.containsKey("tag1")); + Assertions.assertTrue(resultTags1.containsKey("tag3")); + + Assertions.assertTrue(resultTags1.get("tag1").inherited().get()); + Assertions.assertFalse(resultTags1.get("tag3").inherited().get()); + + // Test table tags + Response response8 = + target(tagPath(metalake)) + .path(table.type().toString()) + .path(table.fullName()) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response8.getStatus()); + + TagListResponse tagListResponse2 = response8.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResponse2.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length + tableTagInfos.length, + tagListResponse2.getTags().length); + + Map resultTags2 = + Arrays.stream(tagListResponse2.getTags()) + .collect(Collectors.toMap(Tag::name, Function.identity())); + + Assertions.assertTrue(resultTags2.containsKey("tag1")); + Assertions.assertTrue(resultTags2.containsKey("tag3")); + Assertions.assertTrue(resultTags2.containsKey("tag5")); + + Assertions.assertTrue(resultTags2.get("tag1").inherited().get()); + Assertions.assertTrue(resultTags2.get("tag3").inherited().get()); + Assertions.assertFalse(resultTags2.get("tag5").inherited().get()); + + // Test column tags + Response response9 = + target(tagPath(metalake)) + .path(column.type().toString()) + .path(column.fullName()) + .queryParam("details", true) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response9.getStatus()); + + TagListResponse tagListResponse3 = response9.readEntity(TagListResponse.class); + Assertions.assertEquals(0, tagListResponse3.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + + catalogTagInfos.length + + tableTagInfos.length + + columnTagInfos.length, + tagListResponse3.getTags().length); + + Map resultTags3 = + Arrays.stream(tagListResponse3.getTags()) + .collect(Collectors.toMap(Tag::name, Function.identity())); + + Assertions.assertTrue(resultTags3.containsKey("tag1")); + Assertions.assertTrue(resultTags3.containsKey("tag3")); + Assertions.assertTrue(resultTags3.containsKey("tag5")); + Assertions.assertTrue(resultTags3.containsKey("tag7")); + + Assertions.assertTrue(resultTags3.get("tag1").inherited().get()); + Assertions.assertTrue(resultTags3.get("tag3").inherited().get()); + Assertions.assertTrue(resultTags3.get("tag5").inherited().get()); + Assertions.assertFalse(resultTags3.get("tag7").inherited().get()); + } + + @Test + public void testGetTagForObject() { + TagEntity tag1 = + TagEntity.builder().withName("tag1").withId(1L).withAuditInfo(testAuditInfo1).build(); + MetadataObject catalog = MetadataObjects.parse("object1", MetadataObject.Type.CATALOG); + when(tagManager.getTagForMetadataObject(metalake, catalog, "tag1")).thenReturn(tag1); + + TagEntity tag2 = + TagEntity.builder().withName("tag2").withId(1L).withAuditInfo(testAuditInfo1).build(); + MetadataObject schema = MetadataObjects.parse("object1.object2", MetadataObject.Type.SCHEMA); + when(tagManager.getTagForMetadataObject(metalake, schema, "tag2")).thenReturn(tag2); + + TagEntity tag3 = + TagEntity.builder().withName("tag3").withId(1L).withAuditInfo(testAuditInfo1).build(); + MetadataObject table = + MetadataObjects.parse("object1.object2.object3", MetadataObject.Type.TABLE); + when(tagManager.getTagForMetadataObject(metalake, table, "tag3")).thenReturn(tag3); + + TagEntity tag4 = + TagEntity.builder().withName("tag4").withId(1L).withAuditInfo(testAuditInfo1).build(); + MetadataObject column = + MetadataObjects.parse("object1.object2.object3.object4", MetadataObject.Type.COLUMN); + when(tagManager.getTagForMetadataObject(metalake, column, "tag4")).thenReturn(tag4); + + // Test catalog tag + Response response = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + + TagResponse tagResponse = response.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResponse.getCode()); + + Tag respTag = tagResponse.getTag(); + Assertions.assertEquals(tag1.name(), respTag.name()); + Assertions.assertEquals(tag1.comment(), respTag.comment()); + Assertions.assertFalse(respTag.inherited().get()); + + // Test schema tag + Response response1 = + target(tagPath(metalake)) + .path(schema.type().toString()) + .path(schema.fullName()) + .path("tag2") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response1.getStatus()); + + TagResponse tagResponse1 = response1.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResponse1.getCode()); + + Tag respTag1 = tagResponse1.getTag(); + Assertions.assertEquals(tag2.name(), respTag1.name()); + Assertions.assertEquals(tag2.comment(), respTag1.comment()); + Assertions.assertFalse(respTag1.inherited().get()); + + // Test table tag + Response response2 = + target(tagPath(metalake)) + .path(table.type().toString()) + .path(table.fullName()) + .path("tag3") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response2.getStatus()); + + TagResponse tagResponse2 = response2.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResponse2.getCode()); + + Tag respTag2 = tagResponse2.getTag(); + Assertions.assertEquals(tag3.name(), respTag2.name()); + Assertions.assertEquals(tag3.comment(), respTag2.comment()); + Assertions.assertFalse(respTag2.inherited().get()); + + // Test column tag + Response response3 = + target(tagPath(metalake)) + .path(column.type().toString()) + .path(column.fullName()) + .path("tag4") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response3.getStatus()); + + TagResponse tagResponse3 = response3.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResponse3.getCode()); + + Tag respTag3 = tagResponse3.getTag(); + Assertions.assertEquals(tag4.name(), respTag3.name()); + Assertions.assertEquals(tag4.comment(), respTag3.comment()); + Assertions.assertFalse(respTag3.inherited().get()); + + // Test get schema inherited tag + Response response4 = + target(tagPath(metalake)) + .path(schema.type().toString()) + .path(schema.fullName()) + .path("tag1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response4.getStatus()); + + TagResponse tagResponse4 = response4.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResponse4.getCode()); + + Tag respTag4 = tagResponse4.getTag(); + Assertions.assertEquals(tag1.name(), respTag4.name()); + Assertions.assertEquals(tag1.comment(), respTag4.comment()); + Assertions.assertTrue(respTag4.inherited().get()); + + // Test get table inherited tag + Response response5 = + target(tagPath(metalake)) + .path(table.type().toString()) + .path(table.fullName()) + .path("tag2") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response5.getStatus()); + + TagResponse tagResponse5 = response5.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResponse5.getCode()); + + Tag respTag5 = tagResponse5.getTag(); + Assertions.assertEquals(tag2.name(), respTag5.name()); + Assertions.assertEquals(tag2.comment(), respTag5.comment()); + Assertions.assertTrue(respTag5.inherited().get()); + + // Test get column inherited tag + Response response6 = + target(tagPath(metalake)) + .path(column.type().toString()) + .path(column.fullName()) + .path("tag3") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response6.getStatus()); + + TagResponse tagResponse6 = response6.readEntity(TagResponse.class); + Assertions.assertEquals(0, tagResponse6.getCode()); + + Tag respTag6 = tagResponse6.getTag(); + Assertions.assertEquals(tag3.name(), respTag6.name()); + Assertions.assertEquals(tag3.comment(), respTag6.comment()); + Assertions.assertTrue(respTag6.inherited().get()); + + // Test catalog tag throw NoSuchTagException + Response response7 = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .path("tag2") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response7.getStatus()); + + ErrorResponse errorResponse = response7.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchTagException.class.getSimpleName(), errorResponse.getType()); + + // Test schema tag throw NoSuchTagException + Response response8 = + target(tagPath(metalake)) + .path(schema.type().toString()) + .path(schema.fullName()) + .path("tag3") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response8.getStatus()); + + ErrorResponse errorResponse1 = response8.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse1.getCode()); + Assertions.assertEquals(NoSuchTagException.class.getSimpleName(), errorResponse1.getType()); + } + + @Test + public void testAssociateTagsForObject() { + String[] tagsToAdd = new String[] {"tag1", "tag2"}; + String[] tagsToRemove = new String[] {"tag3", "tag4"}; + + MetadataObject catalog = MetadataObjects.parse("object1", MetadataObject.Type.CATALOG); + when(tagManager.associateTagsForMetadataObject(metalake, catalog, tagsToAdd, tagsToRemove)) + .thenReturn(tagsToAdd); + + TagsAssociateRequest request = new TagsAssociateRequest(tagsToAdd, tagsToRemove); + Response response = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + NameListResponse nameListResponse = response.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse.getCode()); + + Assertions.assertArrayEquals(tagsToAdd, nameListResponse.getNames()); + + // Test throw null tags + when(tagManager.associateTagsForMetadataObject(metalake, catalog, tagsToAdd, tagsToRemove)) + .thenReturn(null); + Response response1 = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response1.getStatus()); + + NameListResponse nameListResponse1 = response1.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse1.getCode()); + + Assertions.assertEquals(0, nameListResponse1.getNames().length); + + // Test throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(tagManager) + .associateTagsForMetadataObject(any(), any(), any(), any()); + + Response response2 = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), response2.getStatus()); + + ErrorResponse errorResponse = response2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); + } + + private String tagPath(String metalake) { + return "/metalakes/" + metalake + "/tags"; + } +} From 5e89f9c73288eaebc41969aa750581719703fabf Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Fri, 12 Jul 2024 19:36:53 +0800 Subject: [PATCH 19/27] Rebase and fix the issues --- .../dto/requests/TagCreateRequest.java | 18 ++- .../dto/requests/TagUpdateRequest.java | 18 ++- .../dto/requests/TagUpdatesRequest.java | 18 ++- .../dto/requests/TagsAssociateRequest.java | 18 ++- .../dto/responses/NameListResponse.java | 19 ++- .../dto/responses/TagListResponse.java | 18 ++- .../gravitino/dto/responses/TagResponse.java | 18 ++- .../gravitino/dto/tag/MetadataObjectDTO.java | 19 ++- .../datastrato/gravitino/dto/tag/TagDTO.java | 46 +++---- .../gravitino/dto/util/DTOConverters.java | 7 -- .../dto/requests/TestTagCreateRequest.java | 19 ++- .../dto/requests/TestTagUpdatesRequest.java | 19 ++- .../dto/tag/TestMetadataObjectDTO.java | 19 ++- .../gravitino/dto/tag/TestTagDTO.java | 63 +++------- .../datastrato/gravitino/tag/TagManager.java | 7 +- .../server/web/rest/TagOperations.java | 43 ++++--- .../server/web/rest/TestTagOperations.java | 117 ++++-------------- 17 files changed, 259 insertions(+), 227 deletions(-) diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java index 8452162d1bb..9a902949c74 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagCreateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java index 5f058a393ff..03ff9b9662a 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java index f614ed4a754..07176300dfa 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagUpdatesRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java index 14988303c26..560949a2e11 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TagsAssociateRequest.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.requests; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java index adcd733b0df..8b98844c5de 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/NameListResponse.java @@ -1,8 +1,21 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package com.datastrato.gravitino.dto.responses; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java index dbc62e73d17..bec465cda41 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagListResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java index fdcb68ed5c9..281a4e79e02 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/TagResponse.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.dto.responses; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java index 986fd30e6b9..e178e00ccf1 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/tag/MetadataObjectDTO.java @@ -1,8 +1,21 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package com.datastrato.gravitino.dto.tag; import com.datastrato.gravitino.MetadataObject; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java index f4e2621c6a4..de0400ee06e 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/tag/TagDTO.java @@ -1,11 +1,23 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package com.datastrato.gravitino.dto.tag; -import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.tag.Tag; import com.fasterxml.jackson.annotation.JsonProperty; @@ -15,7 +27,7 @@ /** Represents a Tag Data Transfer Object (DTO). */ @EqualsAndHashCode -public class TagDTO implements Tag, Tag.AssociatedObjects { +public class TagDTO implements Tag { @JsonProperty("name") private String name; @@ -32,9 +44,6 @@ public class TagDTO implements Tag, Tag.AssociatedObjects { @JsonProperty("inherited") private Optional inherited = Optional.empty(); - @JsonProperty("objects") - private MetadataObjectDTO[] objects; - private TagDTO() {} @Override @@ -62,16 +71,6 @@ public Optional inherited() { return inherited; } - @Override - public MetadataObject[] objects() { - return objects; - } - - @Override - public AssociatedObjects associatedObjects() { - return this; - } - /** @return a new builder for constructing a Tag DTO. */ public static Builder builder() { return new Builder(); @@ -140,17 +139,6 @@ public Builder withInherited(Optional inherited) { return this; } - /** - * Sets the objects associated with the tag. - * - * @param objects The objects associated with the tag. - * @return The builder instance. - */ - public Builder withObjects(MetadataObjectDTO[] objects) { - tagDTO.objects = objects; - return this; - } - /** @return The constructed Tag DTO. */ public TagDTO build() { return tagDTO; diff --git a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java index c9053f5610d..67216865174 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/util/DTOConverters.java @@ -500,13 +500,6 @@ public static TagDTO toDTO(Tag tag, Optional inherited) { .withAudit(toDTO(tag.auditInfo())) .withInherited(inherited); - Optional.ofNullable(tag.associatedObjects().objects()) - .map(Arrays::stream) - .ifPresent( - objects -> - builder.withObjects( - objects.map(DTOConverters::toDTO).toArray(MetadataObjectDTO[]::new))); - return builder.build(); } diff --git a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java index 5473a8fafb1..9bd21424dc4 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagCreateRequest.java @@ -1,8 +1,21 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package com.datastrato.gravitino.dto.requests; import com.datastrato.gravitino.json.JsonUtils; diff --git a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java index ab87acc46f3..8b32925f929 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTagUpdatesRequest.java @@ -1,8 +1,21 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package com.datastrato.gravitino.dto.requests; import com.datastrato.gravitino.json.JsonUtils; diff --git a/common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java index 7c929e07f94..d93c89e5486 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestMetadataObjectDTO.java @@ -1,8 +1,21 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package com.datastrato.gravitino.dto.tag; import com.datastrato.gravitino.MetadataObject; diff --git a/common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java index d93755e1c45..875e873247c 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/tag/TestTagDTO.java @@ -1,11 +1,23 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ - package com.datastrato.gravitino.dto.tag; -import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.dto.AuditDTO; import com.datastrato.gravitino.json.JsonUtils; import com.fasterxml.jackson.core.JsonProcessingException; @@ -87,48 +99,5 @@ public void testTagSerDe() throws JsonProcessingException { serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO4); TagDTO deserTagDTO4 = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); Assertions.assertEquals(Optional.of(true), deserTagDTO4.inherited()); - - // Test tag with metadata objects - MetadataObjectDTO metalakeDTO = - MetadataObjectDTO.builder() - .withName("metalake_test") - .withType(MetadataObject.Type.METALAKE) - .build(); - - MetadataObjectDTO catalogDTO = - MetadataObjectDTO.builder() - .withName("catalog_test") - .withType(MetadataObject.Type.CATALOG) - .build(); - - MetadataObjectDTO schemaDTO = - MetadataObjectDTO.builder() - .withName("schema_test") - .withParent("catalog_test") - .withType(MetadataObject.Type.SCHEMA) - .build(); - - MetadataObjectDTO tableDTO = - MetadataObjectDTO.builder() - .withName("table_test") - .withParent("catalog_test.schema_test") - .withType(MetadataObject.Type.TABLE) - .build(); - - MetadataObjectDTO[] objectDTOs = - new MetadataObjectDTO[] {metalakeDTO, catalogDTO, schemaDTO, tableDTO}; - - TagDTO tagDTO5 = - TagDTO.builder() - .withName("tag_test") - .withComment("tag comment") - .withAudit(audit) - .withObjects(objectDTOs) - .build(); - - serJson = JsonUtils.objectMapper().writeValueAsString(tagDTO5); - TagDTO deserTagDTO5 = JsonUtils.objectMapper().readValue(serJson, TagDTO.class); - Assertions.assertEquals(tagDTO5, deserTagDTO5); - Assertions.assertArrayEquals(objectDTOs, deserTagDTO5.objects()); } } diff --git a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java index 1cf78a25c56..a9b052ed993 100644 --- a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java +++ b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java @@ -85,6 +85,10 @@ public TagManager(IdGenerator idGenerator, EntityStore entityStore) { } public String[] listTags(String metalake) { + return Arrays.stream(listTagsInfo(metalake)).map(Tag::name).toArray(String[]::new); + } + + public Tag[] listTagsInfo(String metalake) { return TreeLockUtils.doWithTreeLock( NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.READ, @@ -94,8 +98,7 @@ public String[] listTags(String metalake) { try { return entityStore .list(ofTagNamespace(metalake), TagEntity.class, Entity.EntityType.TAG).stream() - .map(TagEntity::name) - .toArray(String[]::new); + .toArray(Tag[]::new); } catch (IOException ioe) { LOG.error("Failed to list tags under metalake {}", metalake, ioe); throw new RuntimeException(ioe); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java index 9c0a460944c..90bcfb2adc8 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.server.web.rest; @@ -66,20 +80,16 @@ public TagOperations(TagManager tagManager) { @ResponseMetered(name = "list-tags", absolute = true) public Response listTags( @PathParam("metalake") String metalake, - @QueryParam("details") @DefaultValue("false") boolean verbose, - @QueryParam("extended") @DefaultValue("false") boolean extended) { + @QueryParam("details") @DefaultValue("false") boolean verbose) { LOG.info( - "Received list tag {} with extended {} request for metalake: {}", - verbose ? "infos" : "names", - extended, - metalake); + "Received list tag {} request for metalake: {}", verbose ? "infos" : "names", metalake); try { return Utils.doAs( httpRequest, () -> { if (verbose) { - Tag[] tags = tagManager.listTagsInfo(metalake, extended); + Tag[] tags = tagManager.listTagsInfo(metalake); TagDTO[] tagDTOs; if (ArrayUtils.isEmpty(tags)) { tagDTOs = new TagDTO[0]; @@ -90,11 +100,7 @@ public Response listTags( .toArray(TagDTO[]::new); } - LOG.info( - "List {} tags info with extended {} under metalake: {}", - tagDTOs.length, - extended, - metalake); + LOG.info("List {} tags info under metalake: {}", tagDTOs.length, metalake); return Utils.ok(new TagListResponse(tagDTOs)); } else { @@ -140,17 +146,14 @@ public Response createTag(@PathParam("metalake") String metalake, TagCreateReque @Produces("application/vnd.gravitino.v1+json") @Timed(name = "get-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "get-tag", absolute = true) - public Response getTag( - @PathParam("metalake") String metalake, - @PathParam("tag") String name, - @QueryParam("extended") @DefaultValue("false") boolean extended) { + public Response getTag(@PathParam("metalake") String metalake, @PathParam("tag") String name) { LOG.info("Received get tag request for tag: {} under metalake: {}", name, metalake); try { return Utils.doAs( httpRequest, () -> { - Tag tag = tagManager.getTag(metalake, name, extended); + Tag tag = tagManager.getTag(metalake, name); LOG.info("Get tag: {} under metalake: {}", name, metalake); return Utils.ok(new TagResponse(DTOConverters.toDTO(tag, Optional.empty()))); }); @@ -221,7 +224,7 @@ public Response deleteTag(@PathParam("metalake") String metalake, @PathParam("ta @Produces("application/vnd.gravitino.v1+json") @Timed(name = "list-object-tags." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "list-object-tags", absolute = true) - public Response listTagsForObject( + public Response listTagsForMetdataObject( @PathParam("metalake") String metalake, @PathParam("type") String type, @PathParam("fullName") String fullName, diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java index 7da6e8f2f0e..03ca0f746e8 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java @@ -1,6 +1,20 @@ /* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.datastrato.gravitino.server.web.rest; @@ -185,11 +199,10 @@ public void testListTagsInfo() { .build(); Tag[] tags = new Tag[] {tag1, tag2}; - when(tagManager.listTagsInfo(metalake, false)).thenReturn(tags); + when(tagManager.listTagsInfo(metalake)).thenReturn(tags); Response resp = target(tagPath(metalake)) - .queryParam("extended", false) .queryParam("details", true) .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") @@ -211,10 +224,9 @@ public void testListTagsInfo() { Assertions.assertEquals(Optional.empty(), tagListResp.getTags()[1].inherited()); // Test return null - when(tagManager.listTagsInfo(metalake, false)).thenReturn(null); + when(tagManager.listTagsInfo(metalake)).thenReturn(null); Response resp1 = target(tagPath(metalake)) - .queryParam("extended", false) .queryParam("details", true) .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") @@ -227,10 +239,9 @@ public void testListTagsInfo() { Assertions.assertEquals(0, tagListResp1.getTags().length); // Test return empty array - when(tagManager.listTagsInfo(metalake, false)).thenReturn(new Tag[0]); + when(tagManager.listTagsInfo(metalake)).thenReturn(new Tag[0]); Response resp2 = target(tagPath(metalake)) - .queryParam("extended", false) .queryParam("details", true) .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") @@ -241,55 +252,6 @@ public void testListTagsInfo() { TagListResponse tagListResp2 = resp2.readEntity(TagListResponse.class); Assertions.assertEquals(0, tagListResp2.getCode()); Assertions.assertEquals(0, tagListResp2.getTags().length); - - // Test with extended = true - MetadataObject[] objects = - new MetadataObject[] {MetadataObjects.parse("catalog_1", MetadataObject.Type.CATALOG)}; - - TagEntity tag3 = - TagEntity.builder() - .withName("tag3") - .withId(1L) - .withComment("tag3 comment") - .withAuditInfo(testAuditInfo1) - .withMetadataObjects(objects) - .build(); - - TagEntity tag4 = - TagEntity.builder() - .withName("tag4") - .withId(1L) - .withComment("tag4 comment") - .withAuditInfo(testAuditInfo1) - .withMetadataObjects(objects) - .build(); - - Tag[] tags1 = new Tag[] {tag3, tag4}; - when(tagManager.listTagsInfo(metalake, true)).thenReturn(tags1); - - Response resp3 = - target(tagPath(metalake)) - .queryParam("extended", true) - .queryParam("details", true) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); - - TagListResponse tagListResp3 = resp3.readEntity(TagListResponse.class); - Assertions.assertEquals(0, tagListResp3.getCode()); - Assertions.assertEquals(tags1.length, tagListResp3.getTags().length); - - Assertions.assertEquals(tag3.name(), tagListResp3.getTags()[0].name()); - Assertions.assertEquals(tag3.comment(), tagListResp3.getTags()[0].comment()); - Assertions.assertEquals(tag3.objects().length, tagListResp3.getTags()[0].objects().length); - Assertions.assertEquals(Optional.empty(), tagListResp3.getTags()[0].inherited()); - - Assertions.assertEquals(tag4.name(), tagListResp3.getTags()[1].name()); - Assertions.assertEquals(tag4.comment(), tagListResp3.getTags()[1].comment()); - Assertions.assertEquals(tag4.objects().length, tagListResp3.getTags()[1].objects().length); - Assertions.assertEquals(Optional.empty(), tagListResp3.getTags()[1].inherited()); } @Test @@ -320,8 +282,6 @@ public void testCreateTag() { Assertions.assertEquals(tag1.name(), respTag.name()); Assertions.assertEquals(tag1.comment(), respTag.comment()); Assertions.assertEquals(Optional.empty(), respTag.inherited()); - Assertions.assertNull(respTag.associatedObjects().objects()); - Assertions.assertEquals(0, respTag.associatedObjects().count()); // Test throw TagAlreadyExistsException doThrow(new TagAlreadyExistsException("mock error")) @@ -367,7 +327,7 @@ public void testGetTag() { .withComment("tag1 comment") .withAuditInfo(testAuditInfo1) .build(); - when(tagManager.getTag(metalake, "tag1", false)).thenReturn(tag1); + when(tagManager.getTag(metalake, "tag1")).thenReturn(tag1); Response resp = target(tagPath(metalake)) @@ -387,41 +347,8 @@ public void testGetTag() { Assertions.assertEquals(tag1.comment(), respTag.comment()); Assertions.assertEquals(Optional.empty(), respTag.inherited()); - // Test get tag with extended = true - MetadataObject[] objects = - new MetadataObject[] {MetadataObjects.parse("catalog_1", MetadataObject.Type.CATALOG)}; - - TagEntity tag2 = - TagEntity.builder() - .withName("tag2") - .withId(1L) - .withComment("tag2 comment") - .withAuditInfo(testAuditInfo1) - .withMetadataObjects(objects) - .build(); - when(tagManager.getTag(metalake, "tag2", true)).thenReturn(tag2); - - Response resp1 = - target(tagPath(metalake)) - .path("tag2") - .queryParam("extended", true) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); - - TagResponse tagResp1 = resp1.readEntity(TagResponse.class); - Assertions.assertEquals(0, tagResp1.getCode()); - - Tag respTag1 = tagResp1.getTag(); - Assertions.assertEquals(tag2.name(), respTag1.name()); - Assertions.assertEquals(tag2.comment(), respTag1.comment()); - Assertions.assertEquals(tag2.objects().length, respTag1.associatedObjects().objects().length); - Assertions.assertEquals(Optional.empty(), respTag1.inherited()); - // Test throw NoSuchTagException - doThrow(new NoSuchTagException("mock error")).when(tagManager).getTag(metalake, "tag1", false); + doThrow(new NoSuchTagException("mock error")).when(tagManager).getTag(metalake, "tag1"); Response resp2 = target(tagPath(metalake)) @@ -437,7 +364,7 @@ public void testGetTag() { Assertions.assertEquals(NoSuchTagException.class.getSimpleName(), errorResp.getType()); // Test throw RuntimeException - doThrow(new RuntimeException("mock error")).when(tagManager).getTag(metalake, "tag1", false); + doThrow(new RuntimeException("mock error")).when(tagManager).getTag(metalake, "tag1"); Response resp3 = target(tagPath(metalake)) From 5918f62427bdf55364ccb55a89666cf5d819bb58 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 15 Jul 2024 21:03:07 +0800 Subject: [PATCH 20/27] Rebase and fix the issues --- .../responses/MetadataObjectListResponse.java | 71 ++++++++++++ .../datastrato/gravitino/tag/TagManager.java | 2 - .../server/web/rest/ExceptionHandlers.java | 4 + .../server/web/rest/TagOperations.java | 34 ++++++ .../server/web/rest/TestTagOperations.java | 104 +++++++++++++++++- 5 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/responses/MetadataObjectListResponse.java diff --git a/common/src/main/java/com/datastrato/gravitino/dto/responses/MetadataObjectListResponse.java b/common/src/main/java/com/datastrato/gravitino/dto/responses/MetadataObjectListResponse.java new file mode 100644 index 00000000000..f21954848e7 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/responses/MetadataObjectListResponse.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.datastrato.gravitino.dto.responses; + +import com.datastrato.gravitino.dto.tag.MetadataObjectDTO; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; + +/** Represents a response containing a list of metadata objects. */ +@Getter +@EqualsAndHashCode(callSuper = true) +@ToString +public class MetadataObjectListResponse extends BaseResponse { + + @JsonProperty("metadataObjects") + private final MetadataObjectDTO[] metadataObjects; + + /** + * Constructor for MetadataObjectListResponse. + * + * @param metadataObjects The array of metadata object DTOs. + */ + public MetadataObjectListResponse(MetadataObjectDTO[] metadataObjects) { + super(0); + this.metadataObjects = metadataObjects; + } + + /** Default constructor for MetadataObjectListResponse. (Used for Jackson deserialization.) */ + public MetadataObjectListResponse() { + super(); + this.metadataObjects = null; + } + + /** + * Validates the response data. + * + * @throws IllegalArgumentException if name or audit information is not set. + */ + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(metadataObjects != null, "metadataObjects must be non-null"); + Arrays.stream(metadataObjects) + .forEach( + object -> + Preconditions.checkArgument(object != null && StringUtils.isNoneBlank(object.name()), + "metadataObject must not be null and empty")); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java index a9b052ed993..9f6148f8204 100644 --- a/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java +++ b/core/src/main/java/com/datastrato/gravitino/tag/TagManager.java @@ -313,8 +313,6 @@ public String[] associateTagsForMetadataObject( .map(tag -> ofTagIdent(metalake, tag)) .toArray(NameIdentifier[]::new); - // TODO. We need to add a write lock to Tag's namespace to avoid tag alteration and deletion - // during the association operation. return TreeLockUtils.doWithTreeLock( entityIdent, LockType.READ, diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java index 69562a6475d..f9a294784dc 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/ExceptionHandlers.java @@ -29,6 +29,7 @@ import com.datastrato.gravitino.exceptions.RoleAlreadyExistsException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.exceptions.TagAlreadyAssociatedException; import com.datastrato.gravitino.exceptions.TagAlreadyExistsException; import com.datastrato.gravitino.exceptions.TopicAlreadyExistsException; import com.datastrato.gravitino.exceptions.UserAlreadyExistsException; @@ -520,6 +521,9 @@ public Response handle(OperationType op, String tag, String parent, Exception e) } else if (e instanceof TagAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof TagAlreadyAssociatedException) { + return Utils.alreadyExists(errorMsg, e); + } else { return super.handle(op, tag, parent, e); } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java index 90bcfb2adc8..6a9b571cf30 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TagOperations.java @@ -27,9 +27,11 @@ import com.datastrato.gravitino.dto.requests.TagUpdatesRequest; import com.datastrato.gravitino.dto.requests.TagsAssociateRequest; import com.datastrato.gravitino.dto.responses.DropResponse; +import com.datastrato.gravitino.dto.responses.MetadataObjectListResponse; import com.datastrato.gravitino.dto.responses.NameListResponse; import com.datastrato.gravitino.dto.responses.TagListResponse; import com.datastrato.gravitino.dto.responses.TagResponse; +import com.datastrato.gravitino.dto.tag.MetadataObjectDTO; import com.datastrato.gravitino.dto.tag.TagDTO; import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.exceptions.NoSuchTagException; @@ -373,6 +375,38 @@ public Response getTagForObject( } } + @GET + @Path("{tag}/objects") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-objects-for-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-objects-for-tag", absolute = true) + public Response listMetadataObjectsForTag( + @PathParam("metalake") String metalake, @PathParam("tag") String tagName) { + LOG.info("Received list objects for tag: {} under metalake: {}", tagName, metalake); + + try { + return Utils.doAs( + httpRequest, + () -> { + MetadataObject[] objects = tagManager.listMetadataObjectsForTag(metalake, tagName); + objects = objects == null ? new MetadataObject[0] : objects; + + LOG.info( + "List {} objects for tag: {} under metalake: {}", + objects.length, + tagName, + metalake); + + MetadataObjectDTO[] objectDTOs = + Arrays.stream(objects).map(DTOConverters::toDTO).toArray(MetadataObjectDTO[]::new); + return Utils.ok(new MetadataObjectListResponse(objectDTOs)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleTagException(OperationType.LIST, "", tagName, e); + } + } + @POST @Path("{type}/{fullName}") @Produces("application/vnd.gravitino.v1+json") diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java index 03ca0f746e8..038f505f6f3 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTagOperations.java @@ -32,11 +32,13 @@ import com.datastrato.gravitino.dto.responses.DropResponse; import com.datastrato.gravitino.dto.responses.ErrorConstants; import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.dto.responses.MetadataObjectListResponse; import com.datastrato.gravitino.dto.responses.NameListResponse; import com.datastrato.gravitino.dto.responses.TagListResponse; import com.datastrato.gravitino.dto.responses.TagResponse; import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.exceptions.NoSuchTagException; +import com.datastrato.gravitino.exceptions.TagAlreadyAssociatedException; import com.datastrato.gravitino.exceptions.TagAlreadyExistsException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.TagEntity; @@ -1012,12 +1014,31 @@ public void testAssociateTagsForObject() { Assertions.assertEquals(0, nameListResponse1.getNames().length); + // Test throw TagAlreadyAssociatedException + doThrow(new TagAlreadyAssociatedException("mock error")) + .when(tagManager) + .associateTagsForMetadataObject(metalake, catalog, tagsToAdd, tagsToRemove); + Response response2 = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), response2.getStatus()); + + ErrorResponse errorResponse = response2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResponse.getCode()); + Assertions.assertEquals( + TagAlreadyAssociatedException.class.getSimpleName(), errorResponse.getType()); + // Test throw RuntimeException doThrow(new RuntimeException("mock error")) .when(tagManager) .associateTagsForMetadataObject(any(), any(), any(), any()); - Response response2 = + Response response3 = target(tagPath(metalake)) .path(catalog.type().toString()) .path(catalog.fullName()) @@ -1025,12 +1046,87 @@ public void testAssociateTagsForObject() { .accept("application/vnd.gravitino.v1+json") .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), response3.getStatus()); + + ErrorResponse errorResponse1 = response3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse1.getType()); + } + + @Test + public void testListMetadataObjectForTag() { + MetadataObject[] objects = + new MetadataObject[] { + MetadataObjects.parse("object1", MetadataObject.Type.CATALOG), + MetadataObjects.parse("object1.object2", MetadataObject.Type.SCHEMA), + MetadataObjects.parse("object1.object2.object3", MetadataObject.Type.TABLE), + MetadataObjects.parse("object1.object2.object3.object4", MetadataObject.Type.COLUMN) + }; + + when(tagManager.listMetadataObjectsForTag(metalake, "tag1")).thenReturn(objects); + + Response response = + target(tagPath(metalake)) + .path("tag1") + .path("objects") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + MetadataObjectListResponse objectListResponse = + response.readEntity(MetadataObjectListResponse.class); + Assertions.assertEquals(0, objectListResponse.getCode()); + + MetadataObject[] respObjects = objectListResponse.getMetadataObjects(); + Assertions.assertEquals(objects.length, respObjects.length); + + for (int i = 0; i < objects.length; i++) { + Assertions.assertEquals(objects[i].type(), respObjects[i].type()); + Assertions.assertEquals(objects[i].fullName(), respObjects[i].fullName()); + } + + // Test throw NoSuchTagException + doThrow(new NoSuchTagException("mock error")) + .when(tagManager) + .listMetadataObjectsForTag(metalake, "tag1"); + + Response response1 = + target(tagPath(metalake)) + .path("tag1") + .path("objects") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response1.getStatus()); + + ErrorResponse errorResponse = response1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); + Assertions.assertEquals(NoSuchTagException.class.getSimpleName(), errorResponse.getType()); + + // Test throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(tagManager) + .listMetadataObjectsForTag(any(), any()); + + Response response2 = + target(tagPath(metalake)) + .path("tag1") + .path("objects") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals( Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), response2.getStatus()); - ErrorResponse errorResponse = response2.readEntity(ErrorResponse.class); - Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse.getCode()); - Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); + ErrorResponse errorResponse1 = response2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse1.getType()); } private String tagPath(String metalake) { From 0f2eaf26cd13fbe205ee91a6ad9143a521bb373c Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 16 Jul 2024 11:20:46 +0800 Subject: [PATCH 21/27] Add core logic for tag management (part-2) --- .../TagAlreadyAssociatedException.java | 49 +++ .../org/apache/gravitino/EntityStore.java | 11 + .../storage/relational/JDBCBackend.java | 32 ++ .../storage/relational/RelationalBackend.java | 3 +- .../relational/RelationalEntityStore.java | 41 +- .../relational/mapper/TagMetaMapper.java | 33 +- .../mapper/TagMetadataObjectRelMapper.java | 113 +++++- .../relational/po/TagMetadataObjectRelPO.java | 130 +++++++ .../storage/relational/po/TagPO.java | 21 +- .../storage/relational/po/TopicPO.java | 26 +- .../MetadataObjectService.java} | 8 +- .../relational/service/RoleMetaService.java | 5 +- .../relational/service/TagMetaService.java | 198 ++++++++++ .../relational/utils/POConverters.java | 26 ++ .../gravitino/tag/SupportsTagOperations.java | 98 +++++ .../org/apache/gravitino/tag/TagManager.java | 165 +++++++- .../gravitino/utils/MetadataObjectUtil.java | 90 +++++ .../gravitino/utils/NameIdentifierUtil.java | 55 +++ .../service/TestTagMetaService.java | 362 ++++++++++++++++++ .../relational/utils/TestPOConverters.java | 17 + .../apache/gravitino/tag/TestTagManager.java | 346 +++++++++++++++++ .../utils/TestNameIdentifierUtil.java | 66 ++++ scripts/h2/schema-h2.sql | 2 +- scripts/mysql/schema-0.6.0-mysql.sql | 2 +- .../mysql/upgrade-0.5.0-to-0.6.0-mysql.sql | 2 +- 25 files changed, 1844 insertions(+), 57 deletions(-) create mode 100644 api/src/main/java/org/apache/gravitino/exceptions/TagAlreadyAssociatedException.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/po/TagMetadataObjectRelPO.java rename core/src/main/java/org/apache/gravitino/storage/relational/{utils/MetadataObjectUtils.java => service/MetadataObjectService.java} (96%) create mode 100644 core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java create mode 100644 core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java diff --git a/api/src/main/java/org/apache/gravitino/exceptions/TagAlreadyAssociatedException.java b/api/src/main/java/org/apache/gravitino/exceptions/TagAlreadyAssociatedException.java new file mode 100644 index 00000000000..61cab11fb49 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/TagAlreadyAssociatedException.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; + +/** Exception thrown when a tag with specified name already associated to a metadata object. */ +public class TagAlreadyAssociatedException extends AlreadyExistsException { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public TagAlreadyAssociatedException(String message, Object... args) { + super(message, args); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public TagAlreadyAssociatedException(Throwable cause, String message, Object... args) { + super(cause, message, args); + } +} diff --git a/core/src/main/java/org/apache/gravitino/EntityStore.java b/core/src/main/java/org/apache/gravitino/EntityStore.java index ec9eab8d40a..7831e510e3d 100644 --- a/core/src/main/java/org/apache/gravitino/EntityStore.java +++ b/core/src/main/java/org/apache/gravitino/EntityStore.java @@ -24,6 +24,7 @@ import java.util.function.Function; import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.tag.SupportsTagOperations; import org.apache.gravitino.utils.Executable; public interface EntityStore extends Closeable { @@ -183,4 +184,14 @@ default boolean delete(NameIdentifier ident, EntityType entityType) throws IOExc */ R executeInTransaction(Executable executable) throws E, IOException; + + /** + * Get the extra tag operations that are supported by the entity store. + * + * @return the tag operations object that are supported by the entity store + * @throws UnsupportedOperationException if the extra operations are not supported + */ + default SupportsTagOperations tagOperations() { + throw new UnsupportedOperationException("extra operations are not supported"); + } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 430fb5a55d5..06029db45e7 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -31,6 +31,7 @@ import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.UnsupportedEntityTypeException; @@ -328,6 +329,37 @@ public void close() throws IOException { } } + @Override + public List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + throws IOException { + return TagMetaService.getInstance().listAssociatedMetadataObjectIdentsForTag(tagIdent); + } + + @Override + public List listAssociatedTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchEntityException, IOException { + return TagMetaService.getInstance().listTagsForMetadataObject(objectIdent, objectType); + } + + @Override + public TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException { + return TagMetaService.getInstance().getTagForMetadataObject(objectIdent, objectType, tagIdent); + } + + @Override + public List associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException { + return TagMetaService.getInstance() + .associateTagsWithMetadataObject(objectIdent, objectType, tagsToAdd, tagsToRemove); + } + enum JDBCBackendType { H2(true), MYSQL(false); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java index 5c81bd56c66..4521a892f8d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalBackend.java @@ -29,9 +29,10 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.tag.SupportsTagOperations; /** Interface defining the operations for a Relation Backend. */ -public interface RelationalBackend extends Closeable { +public interface RelationalBackend extends Closeable, SupportsTagOperations { /** * Initializes the Relational Backend environment with the provided configuration. diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index 25a4290ccc1..9d5c0b21cd3 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -31,9 +31,12 @@ import org.apache.gravitino.EntitySerDe; import org.apache.gravitino.EntityStore; import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.meta.TagEntity; +import org.apache.gravitino.tag.SupportsTagOperations; import org.apache.gravitino.utils.Executable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +46,7 @@ * MySQL, PostgreSQL, etc. If you want to use a different backend, you can implement the {@link * RelationalBackend} interface */ -public class RelationalEntityStore implements EntityStore { +public class RelationalEntityStore implements EntityStore, SupportsTagOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RelationalEntityStore.class); public static final ImmutableMap RELATIONAL_BACKENDS = ImmutableMap.of( @@ -132,4 +135,40 @@ public void close() throws IOException { garbageCollector.close(); backend.close(); } + + @Override + public SupportsTagOperations tagOperations() { + return this; + } + + @Override + public List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + throws IOException { + return backend.listAssociatedMetadataObjectsForTag(tagIdent); + } + + @Override + public List listAssociatedTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchEntityException, IOException { + return backend.listAssociatedTagsForMetadataObject(objectIdent, objectType); + } + + @Override + public TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException { + return backend.getTagForMetadataObject(objectIdent, objectType, tagIdent); + } + + @Override + public List associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException { + return backend.associateTagsWithMetadataObject( + objectIdent, objectType, tagsToAdd, tagsToRemove); + } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java index 44149fc5bb6..273a8771c5f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -31,7 +31,7 @@ public interface TagMetaMapper { String TAG_TABLE_NAME = "tag_meta"; @Select( - "SELECT tm.tag_id as tagId, tag_name as tagName," + "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + " tm.metalake_id as metalakeId," + " tm.tag_comment as comment," + " tm.properties as properties," @@ -43,16 +43,41 @@ public interface TagMetaMapper { + TAG_TABLE_NAME + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME - + " mm on tm.metalake_id = mm.metalake_id" + + " mm ON tm.metalake_id = mm.metalake_id" + " WHERE mm.metalake_name = #{metalakeName} AND tm.deleted_at = 0 AND mm.deleted_at = 0") List listTagPOsByMetalake(@Param("metalakeName") String metalakeName); + @Select( + "") + List listTagPOsByMetalakeAndTagNames( + @Param("metalakeName") String metalakeName, @Param("tagNames") List tagNames); + @Select( "SELECT tm.tag_id as tagId FROM " + TAG_TABLE_NAME + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME - + " mm on tm.metalake_id = mm.metalake_id" + + " mm ON tm.metalake_id = mm.metalake_id" + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") Long selectTagIdByMetalakeAndName( @@ -71,7 +96,7 @@ Long selectTagIdByMetalakeAndName( + TAG_TABLE_NAME + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME - + " mm on tm.metalake_id = mm.metalake_id" + + " mm ON tm.metalake_id = mm.metalake_id" + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + " AND tm.deleted_at = 0 AND mm.deleted_at = 0") TagPO selectTagMetaByMetalakeAndName( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index a6f56971c9c..06fa6e745cd 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -18,36 +18,137 @@ */ package org.apache.gravitino.storage.relational.mapper; +import java.util.List; + +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; +import org.apache.gravitino.storage.relational.po.TagPO; import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Insert; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.Update; public interface TagMetadataObjectRelMapper { String TAG_METADATA_OBJECT_RELATION_TABLE_NAME = "tag_relation_meta"; + @Select( + "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND te.deleted_at = 0" + + " AND tm.deleted_at = 0") + List listTagPOsByMetadataObjectIdAndType( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType); + + @Select( + "SELECT tm.tag_id as tagId, tm.tag_name as tagName," + + " tm.metalake_id as metalakeId, tm.tag_comment as comment, tm.properties as properties," + + " tm.audit_info as auditInfo," + + " tm.current_version as currentVersion," + + " tm.last_version as lastVersion," + + " tm.deleted_at as deletedAt" + + " FROM " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te ON tm.tag_id = te.tag_id" + + " WHERE te.metadata_object_id = #{metadataObjectId}" + + " AND te.metadata_object_type = #{metadataObjectType} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0") + TagPO getTagPOsByMetadataObjectAndTagName( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagName") String tagName); + + @Select( + "SELECT te.tag_id as tagId, te.metadata_object_id as metadataObjectId," + + " te.metadata_object_type as metadataObjectType, te.audit_info as auditInfo," + + " te.current_version as currentVersion, te.last_version as lastVersion," + + " te.deleted_at as deletedAt" + + " FROM " + + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + + " te JOIN " + + TagMetaMapper.TAG_TABLE_NAME + + " tm JOIN " + + MetalakeMetaMapper.TABLE_NAME + + " mm ON te.tag_id = tm.tag_id AND tm.metalake_id = mm.metalake_id" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name = #{tagName}" + + " AND te.deleted_at = 0 AND tm.deleted_at = 0 AND mm.deleted_at = 0") + List listTagMetadataObjectRelsByMetalakeAndTagName( + @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); + + @Insert({ + "" + }) + void batchInsertTagMetadataObjectRels(@Param("tagRels") List tagRelPOs); + + @Update({ + "" + }) + void batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + @Param("metadataObjectId") Long metadataObjectId, + @Param("metadataObjectType") String metadataObjectType, + @Param("tagIds") List tagIds); + @Update( "UPDATE " + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " tmo SET tmo.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE tmo.tag_id IN (SELECT tm.tag_id FROM " + + " WHERE te.tag_id IN (SELECT tm.tag_id FROM " + TagMetaMapper.TAG_TABLE_NAME + " tm WHERE tm.metalake_id IN (SELECT mm.metalake_id FROM " + MetalakeMetaMapper.TABLE_NAME + " mm WHERE mm.metalake_name = #{metalakeName} AND mm.deleted_at = 0)" - + " AND tm.deleted_at = 0) AND tmo.deleted_at = 0") + + " AND tm.deleted_at = 0) AND te.deleted_at = 0") Integer softDeleteTagMetadataObjectRelsByMetalakeAndTagName( @Param("metalakeName") String metalakeName, @Param("tagName") String tagName); @Update( "UPDATE " + TAG_METADATA_OBJECT_RELATION_TABLE_NAME - + " tmo SET tmo.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + + " te SET te.deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" + " WHERE EXISTS (SELECT * FROM " + TagMetaMapper.TAG_TABLE_NAME - + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = tmo.tag_id" - + " AND tm.deleted_at = 0) AND tmo.deleted_at = 0") + + " tm WHERE tm.metalake_id = #{metalakeId} AND tm.tag_id = te.tag_id" + + " AND tm.deleted_at = 0) AND te.deleted_at = 0") void softDeleteTagMetadataObjectRelsByMetalakeId(@Param("metalakeId") Long metalakeId); @Delete( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/po/TagMetadataObjectRelPO.java b/core/src/main/java/org/apache/gravitino/storage/relational/po/TagMetadataObjectRelPO.java new file mode 100644 index 00000000000..e26b5cfb760 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/po/TagMetadataObjectRelPO.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.po; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; + +@Getter +public class TagMetadataObjectRelPO { + private Long tagId; + private Long metadataObjectId; + private String metadataObjectType; + private String auditInfo; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TagMetadataObjectRelPO)) { + return false; + } + TagMetadataObjectRelPO tagRelPO = (TagMetadataObjectRelPO) o; + return Objects.equal(tagId, tagRelPO.tagId) + && Objects.equal(metadataObjectId, tagRelPO.metadataObjectId) + && Objects.equal(metadataObjectType, tagRelPO.metadataObjectType) + && Objects.equal(auditInfo, tagRelPO.auditInfo) + && Objects.equal(currentVersion, tagRelPO.currentVersion) + && Objects.equal(lastVersion, tagRelPO.lastVersion) + && Objects.equal(deletedAt, tagRelPO.deletedAt); + } + + @Override + public int hashCode() { + return Objects.hashCode( + tagId, + metadataObjectId, + metadataObjectType, + auditInfo, + currentVersion, + lastVersion, + deletedAt); + } + + public static class Builder { + private final TagMetadataObjectRelPO tagRelPO; + + private Builder() { + tagRelPO = new TagMetadataObjectRelPO(); + } + + public Builder withTagId(Long tagId) { + tagRelPO.tagId = tagId; + return this; + } + + public Builder withMetadataObjectId(Long metadataObjectId) { + tagRelPO.metadataObjectId = metadataObjectId; + return this; + } + + public Builder withMetadataObjectType(String metadataObjectType) { + tagRelPO.metadataObjectType = metadataObjectType; + return this; + } + + public Builder withAuditInfo(String auditInfo) { + tagRelPO.auditInfo = auditInfo; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + tagRelPO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + tagRelPO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + tagRelPO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(tagRelPO.tagId != null, "Tag id is required"); + Preconditions.checkArgument( + tagRelPO.metadataObjectId != null, "Metadata object id is required"); + Preconditions.checkArgument( + StringUtils.isNotBlank(tagRelPO.metadataObjectType), + "Metadata object type should not be empty"); + Preconditions.checkArgument(tagRelPO.auditInfo != null, "Audit info is required"); + Preconditions.checkArgument(tagRelPO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument(tagRelPO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(tagRelPO.deletedAt != null, "Deleted at is required"); + } + + public TagMetadataObjectRelPO build() { + validate(); + return tagRelPO; + } + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/po/TagPO.java b/core/src/main/java/org/apache/gravitino/storage/relational/po/TagPO.java index c3e6656d61d..1bf873fe1ac 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/po/TagPO.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/po/TagPO.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.storage.relational.po; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; import lombok.Getter; import org.apache.commons.lang3.StringUtils; @@ -47,20 +48,20 @@ public boolean equals(Object o) { return false; } TagPO tagPO = (TagPO) o; - return java.util.Objects.equals(tagId, tagPO.tagId) - && java.util.Objects.equals(tagName, tagPO.tagName) - && java.util.Objects.equals(metalakeId, tagPO.metalakeId) - && java.util.Objects.equals(comment, tagPO.comment) - && java.util.Objects.equals(properties, tagPO.properties) - && java.util.Objects.equals(auditInfo, tagPO.auditInfo) - && java.util.Objects.equals(currentVersion, tagPO.currentVersion) - && java.util.Objects.equals(lastVersion, tagPO.lastVersion) - && java.util.Objects.equals(deletedAt, tagPO.deletedAt); + return Objects.equal(tagId, tagPO.tagId) + && Objects.equal(tagName, tagPO.tagName) + && Objects.equal(metalakeId, tagPO.metalakeId) + && Objects.equal(comment, tagPO.comment) + && Objects.equal(properties, tagPO.properties) + && Objects.equal(auditInfo, tagPO.auditInfo) + && Objects.equal(currentVersion, tagPO.currentVersion) + && Objects.equal(lastVersion, tagPO.lastVersion) + && Objects.equal(deletedAt, tagPO.deletedAt); } @Override public int hashCode() { - return java.util.Objects.hash( + return Objects.hashCode( tagId, tagName, metalakeId, diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/po/TopicPO.java b/core/src/main/java/org/apache/gravitino/storage/relational/po/TopicPO.java index 960ed52f6a9..1b78c86d1ed 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/po/TopicPO.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/po/TopicPO.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.storage.relational.po; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; -import java.util.Objects; import lombok.Getter; @Getter @@ -49,22 +49,22 @@ public boolean equals(Object o) { return false; } TopicPO topicPO = (TopicPO) o; - return Objects.equals(topicId, topicPO.topicId) - && Objects.equals(topicName, topicPO.topicName) - && Objects.equals(metalakeId, topicPO.metalakeId) - && Objects.equals(catalogId, topicPO.catalogId) - && Objects.equals(schemaId, topicPO.schemaId) - && Objects.equals(comment, topicPO.comment) - && Objects.equals(properties, topicPO.properties) - && Objects.equals(auditInfo, topicPO.auditInfo) - && Objects.equals(currentVersion, topicPO.currentVersion) - && Objects.equals(lastVersion, topicPO.lastVersion) - && Objects.equals(deletedAt, topicPO.deletedAt); + return Objects.equal(topicId, topicPO.topicId) + && Objects.equal(topicName, topicPO.topicName) + && Objects.equal(metalakeId, topicPO.metalakeId) + && Objects.equal(catalogId, topicPO.catalogId) + && Objects.equal(schemaId, topicPO.schemaId) + && Objects.equal(comment, topicPO.comment) + && Objects.equal(properties, topicPO.properties) + && Objects.equal(auditInfo, topicPO.auditInfo) + && Objects.equal(currentVersion, topicPO.currentVersion) + && Objects.equal(lastVersion, topicPO.lastVersion) + && Objects.equal(deletedAt, topicPO.deletedAt); } @Override public int hashCode() { - return Objects.hash( + return Objects.hashCode( topicId, topicName, metalakeId, diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/utils/MetadataObjectUtils.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java similarity index 96% rename from core/src/main/java/org/apache/gravitino/storage/relational/utils/MetadataObjectUtils.java rename to core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index dee5f143f8d..b996c52dc11 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/utils/MetadataObjectUtils.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.storage.relational.utils; +package org.apache.gravitino.storage.relational.service; import com.google.common.base.Joiner; import com.google.common.base.Splitter; @@ -39,16 +39,16 @@ import org.apache.gravitino.storage.relational.service.TopicMetaService; /** - * MetadataObjectUtils is used for converting full name to entity id and converting entity id to + * MetadataObjectService is used for converting full name to entity id and converting entity id to * full name. */ -public class MetadataObjectUtils { +public class MetadataObjectService { private static final String DOT = "."; private static final Joiner DOT_JOINER = Joiner.on(DOT); private static final Splitter DOT_SPLITTER = Splitter.on(DOT); - private MetadataObjectUtils() {} + private MetadataObjectService() {} public static long getMetadataObjectId( long metalakeId, String fullName, MetadataObject.Type type) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index baf3e94e574..f4c2cb7d7c5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -36,7 +36,6 @@ import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.po.SecurableObjectPO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; -import org.apache.gravitino.storage.relational.utils.MetadataObjectUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; import org.slf4j.Logger; @@ -108,7 +107,7 @@ public void insertRole(RoleEntity roleEntity, boolean overwritten) throws IOExce POConverters.initializeSecurablePOBuilderWithVersion( roleEntity.id(), object, getEntityType(object)); objectBuilder.withEntityId( - MetadataObjectUtils.getMetadataObjectId(metalakeId, object.fullName(), object.type())); + MetadataObjectService.getMetadataObjectId(metalakeId, object.fullName(), object.type())); securableObjectPOs.add(objectBuilder.build()); } @@ -154,7 +153,7 @@ public RoleEntity getRoleByIdentifier(NameIdentifier identifier) { for (SecurableObjectPO securableObjectPO : securableObjectPOs) { String fullName = - MetadataObjectUtils.getMetadataObjectFullName( + MetadataObjectService.getMetadataObjectFullName( securableObjectPO.getType(), securableObjectPO.getEntityId()); if (fullName != null) { securableObjects.add( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java index 65b68e694ad..8895fab6da0 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java @@ -19,23 +19,33 @@ package org.apache.gravitino.storage.relational.service; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.HasIdentifier; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.exceptions.NoSuchTagException; import org.apache.gravitino.meta.TagEntity; import org.apache.gravitino.storage.relational.mapper.TagMetaMapper; import org.apache.gravitino.storage.relational.mapper.TagMetadataObjectRelMapper; +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.po.TagPO; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; +import org.apache.gravitino.tag.TagManager; +import org.apache.gravitino.utils.NameIdentifierUtil; public class TagMetaService { @@ -144,6 +154,188 @@ public boolean deleteTag(NameIdentifier ident) { return tagDeletedCount[0] + tagMetadataObjectRelDeletedCount[0] > 0; } + public List listTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchTagException, IOException { + MetadataObject metadataObject = NameIdentifierUtil.toMetadataObject(objectIdent, objectType); + String metalake = objectIdent.namespace().level(0); + + List tagPOs = null; + try { + Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); + Long metadataObjectId = + MetadataObjectService.getMetadataObjectId( + metalakeId, metadataObject.fullName(), metadataObject.type()); + + tagPOs = + SessionUtils.doWithoutCommitAndFetchResult( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.listTagPOsByMetadataObjectIdAndType( + metadataObjectId, metadataObject.type().toString())); + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, objectIdent.toString()); + throw e; + } + + return tagPOs.stream() + .map(tagPO -> POConverters.fromTagPO(tagPO, TagManager.ofTagNamespace(metalake))) + .collect(Collectors.toList()); + } + + public TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException { + MetadataObject metadataObject = NameIdentifierUtil.toMetadataObject(objectIdent, objectType); + String metalake = objectIdent.namespace().level(0); + + TagPO tagPO = null; + try { + Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); + Long metadataObjectId = + MetadataObjectService.getMetadataObjectId( + metalakeId, metadataObject.fullName(), metadataObject.type()); + + tagPO = + SessionUtils.getWithoutCommit( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.getTagPOsByMetadataObjectAndTagName( + metadataObjectId, metadataObject.type().toString(), tagIdent.name())); + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, tagIdent.toString()); + throw e; + } + + if (tagPO == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.TAG.name().toLowerCase(), + tagIdent.name()); + } + + return POConverters.fromTagPO(tagPO, TagManager.ofTagNamespace(metalake)); + } + + public List listAssociatedMetadataObjectIdentsForTag(NameIdentifier tagIdent) + throws IOException { + String metalakeName = tagIdent.namespace().level(0); + String tagName = tagIdent.name(); + + try { + List tagMetadataObjectRelPOs = + SessionUtils.doWithCommitAndFetchResult( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.listTagMetadataObjectRelsByMetalakeAndTagName(metalakeName, tagName)); + + List metadataObjects = Lists.newArrayList(); + for (TagMetadataObjectRelPO po : tagMetadataObjectRelPOs) { + String fullName = + MetadataObjectService.getMetadataObjectFullName( + po.getMetadataObjectType(), po.getMetadataObjectId()); + + // Metadata object may be deleted asynchronously when we query the name, so it will return + // null. We should skip this metadata object. + if (fullName == null) { + continue; + } + + MetadataObject.Type type = MetadataObject.Type.valueOf(po.getMetadataObjectType()); + metadataObjects.add(MetadataObjects.parse(fullName, type)); + } + + return metadataObjects; + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, tagIdent.toString()); + throw e; + } + } + + public List associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException { + MetadataObject metadataObject = NameIdentifierUtil.toMetadataObject(objectIdent, objectType); + String metalake = objectIdent.namespace().level(0); + + try { + Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); + Long metadataObjectId = + MetadataObjectService.getMetadataObjectId( + metalakeId, metadataObject.fullName(), metadataObject.type()); + + // Fetch all the tags need to associate with the metadata object. + List tagNamesToAdd = + Arrays.stream(tagsToAdd).map(NameIdentifier::name).collect(Collectors.toList()); + List tagPOsToAdd = + tagNamesToAdd.isEmpty() + ? Collections.emptyList() + : getTagPOsByMetalakeAndNames(metalake, tagNamesToAdd); + + // Fetch all the tags need to remove from the metadata object. + List tagNamesToRemove = + Arrays.stream(tagsToRemove).map(NameIdentifier::name).collect(Collectors.toList()); + List tagPOsToRemove = + tagNamesToRemove.isEmpty() + ? Collections.emptyList() + : getTagPOsByMetalakeAndNames(metalake, tagNamesToRemove); + + SessionUtils.doMultipleWithCommit( + () -> { + // Insert the tag metadata object relations. + if (tagPOsToAdd.isEmpty()) { + return; + } + + List tagRelsToAdd = + tagPOsToAdd.stream() + .map( + tagPO -> + POConverters.initializeTagMetadataObjectRelPOWithVersion( + tagPO.getTagId(), + metadataObjectId, + metadataObject.type().toString())) + .collect(Collectors.toList()); + SessionUtils.doWithoutCommit( + TagMetadataObjectRelMapper.class, + mapper -> mapper.batchInsertTagMetadataObjectRels(tagRelsToAdd)); + }, + () -> { + // Remove the tag metadata object relations. + if (tagPOsToRemove.isEmpty()) { + return; + } + + List tagIdsToRemove = + tagPOsToRemove.stream().map(TagPO::getTagId).collect(Collectors.toList()); + SessionUtils.doWithoutCommit( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.batchDeleteTagMetadataObjectRelsByTagIdsAndMetadataObject( + metadataObjectId, metadataObject.type().toString(), tagIdsToRemove)); + }); + + // Fetch all the tags associated with the metadata object after the operation. + List tagPOs = + SessionUtils.doWithoutCommitAndFetchResult( + TagMetadataObjectRelMapper.class, + mapper -> + mapper.listTagPOsByMetadataObjectIdAndType( + metadataObjectId, metadataObject.type().toString())); + + return tagPOs.stream() + .map(tagPO -> POConverters.fromTagPO(tagPO, TagManager.ofTagNamespace(metalake))) + .collect(Collectors.toList()); + + } catch (RuntimeException e) { + ExceptionUtils.checkSQLException(e, Entity.EntityType.TAG, objectIdent.toString()); + throw e; + } + } + public int deleteTagMetasByLegacyTimeline(long legacyTimeline, int limit) { int[] tagDeletedCount = new int[] {0}; int[] tagMetadataObjectRelDeletedCount = new int[] {0}; @@ -177,4 +369,10 @@ private TagPO getTagPOByMetalakeAndName(String metalakeName, String tagName) { } return tagPO; } + + private List getTagPOsByMetalakeAndNames(String metalakeName, List tagNames) { + return SessionUtils.getWithoutCommit( + TagMetaMapper.class, + mapper -> mapper.listTagPOsByMetalakeAndTagNames(metalakeName, tagNames)); + } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java index 7f0aa198321..490ad611257 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.Lists; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -56,10 +57,12 @@ import org.apache.gravitino.storage.relational.po.SchemaPO; import org.apache.gravitino.storage.relational.po.SecurableObjectPO; import org.apache.gravitino.storage.relational.po.TablePO; +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.po.TagPO; import org.apache.gravitino.storage.relational.po.TopicPO; import org.apache.gravitino.storage.relational.po.UserPO; import org.apache.gravitino.storage.relational.po.UserRoleRelPO; +import org.apache.gravitino.utils.PrincipalUtils; /** POConverters is a utility class to convert PO to Base and vice versa. */ public class POConverters { @@ -1015,4 +1018,27 @@ public static TagPO updateTagPOWithVersion(TagPO oldTagPO, TagEntity newEntity) throw new RuntimeException("Failed to serialize json object:", e); } } + + public static TagMetadataObjectRelPO initializeTagMetadataObjectRelPOWithVersion( + Long tagId, Long metadataObjectId, String metadataObjectType) { + try { + AuditInfo auditInfo = + AuditInfo.builder() + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) + .withCreateTime(Instant.now()) + .build(); + + return TagMetadataObjectRelPO.builder() + .withTagId(tagId) + .withMetadataObjectId(metadataObjectId) + .withMetadataObjectType(metadataObjectType) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(auditInfo)) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } } diff --git a/core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java b/core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java new file mode 100644 index 00000000000..07157d58809 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.tag; + + +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; +import org.apache.gravitino.EntityStore; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.exceptions.NoSuchEntityException; +import org.apache.gravitino.meta.TagEntity; + +import java.io.IOException; +import java.util.List; + +/** + * An interface to support extra tag operations, this interface should be mixed with {@link + * EntityStore} to provide extra operations. + * + *

Any operations that can be done by the entity store should be added here. + */ +public interface SupportsTagOperations { + + /** + * List all the metadata objects that are associated with the given tag. + * + * @param tagIdent The identifier of the tag. + * @return The list of metadata objects associated with the given tag. + * @throws IOException If an error occurs while accessing the entity store. + */ + List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) + throws IOException; + + /** + * List all the tags that are associated with the given metadata object. + * + * @param objectIdent The identifier of the metadata object. + * @param objectType The type of the metadata object. + * @return The list of tags associated with the given metadata object. + * @throws NoSuchEntityException if the metadata object does not exist. + * @throws IOException If an error occurs while accessing the entity store. + */ + List listAssociatedTagsForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType) + throws NoSuchEntityException, IOException; + + /** + * Get the tag with the given identifier that is associated with the given metadata object. + * + * @param objectIdent The identifier of the metadata object. + * @param objectType The type of the metadata object. + * @param tagIdent The identifier of the tag. + * @return The tag associated with the metadata object. + * @throws NoSuchEntityException if the metadata object does not exist or the tag is not + * associated to the metadata object. + * @throws IOException If an error occurs while accessing the entity store. + */ + TagEntity getTagForMetadataObject( + NameIdentifier objectIdent, Entity.EntityType objectType, NameIdentifier tagIdent) + throws NoSuchEntityException, IOException; + + /** + * Associate the given tags with the given metadata object. + * + * @param objectIdent The identifier of the metadata object. + * @param objectType The type of the metadata object. + * @param tagsToAdd The name of tags to associate with the metadata object. + * @param tagsToRemove the name of tags to remove from the metadata object. + * @return The list of tags associated with the metadata object after the operation. + * @throws NoSuchEntityException if the metadata object does not exist. + * @throws EntityAlreadyExistsException if tags already associated with the metadata object. + * @throws IOException If an error occurs while accessing the entity store. + */ + List associateTagsWithMetadataObject( + NameIdentifier objectIdent, + Entity.EntityType objectType, + NameIdentifier[] tagsToAdd, + NameIdentifier[] tagsToRemove) + throws NoSuchEntityException, EntityAlreadyExistsException, IOException; +} diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index 392d7067296..99e3d4fc81b 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -19,11 +19,16 @@ package org.apache.gravitino.tag; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import java.io.IOException; import java.time.Instant; +import java.util.Arrays; import java.util.Collections; import java.util.Map; +import java.util.Set; + import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; @@ -33,6 +38,8 @@ import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchTagException; +import org.apache.gravitino.exceptions.NotFoundException; +import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; import org.apache.gravitino.exceptions.TagAlreadyExistsException; import org.apache.gravitino.lock.LockType; import org.apache.gravitino.lock.TreeLockUtils; @@ -40,6 +47,7 @@ import org.apache.gravitino.meta.TagEntity; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.kv.KvEntityStore; +import org.apache.gravitino.utils.MetadataObjectUtil; import org.apache.gravitino.utils.PrincipalUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +60,8 @@ public class TagManager { private final EntityStore entityStore; + private final SupportsTagOperations supportsTagOperations; + public TagManager(IdGenerator idGenerator, EntityStore entityStore) { if (entityStore instanceof KvEntityStore) { String errorMsg = @@ -61,6 +71,16 @@ public TagManager(IdGenerator idGenerator, EntityStore entityStore) { throw new RuntimeException(errorMsg); } + if (!(entityStore instanceof SupportsTagOperations)) { + String errorMsg = + "TagManager cannot run with entity store that does not support tag operations, " + + "please configure the entity store to use relational entity store and restart the Gravitino server"; + LOG.error(errorMsg); + throw new RuntimeException(errorMsg); + } + + this.supportsTagOperations = entityStore.tagOperations(); + this.idGenerator = idGenerator; this.entityStore = entityStore; } @@ -84,10 +104,6 @@ public String[] listTags(String metalake) { }); } - public MetadataObject[] listAssociatedMetadataObjectsForTag(String metalake, String name) { - throw new UnsupportedOperationException("Not implemented yet"); - } - public Tag createTag(String metalake, String name, String comment, Map properties) throws TagAlreadyExistsException { Map tagProperties = properties == null ? Collections.emptyMap() : properties; @@ -188,22 +204,147 @@ public boolean deleteTag(String metalake, String name) { }); } - public String[] listTagsForMetadataObject(String metalake, MetadataObject metadataObject) { - throw new UnsupportedOperationException("Not implemented yet"); + public MetadataObject[] listMetadataObjectsForTag(String metalake, String name) + throws NoSuchTagException { + NameIdentifier tagId = ofTagIdent(metalake, name); + return TreeLockUtils.doWithTreeLock( + tagId, + LockType.READ, + () -> { + try { + if (!entityStore.exists(tagId, Entity.EntityType.TAG)) { + throw new NoSuchTagException( + "Tag with name %s under metalake %s does not exist", name, metalake); + } + + return supportsTagOperations + .listAssociatedMetadataObjectsForTag(tagId) + .toArray(new MetadataObject[0]); + } catch (IOException e) { + LOG.error("Failed to list metadata objects for tag {}", name, e); + throw new RuntimeException(e); + } + }); } - public Tag[] listTagsInfoForMetadataObject(String metalake, MetadataObject metadataObject) { - throw new UnsupportedOperationException("Not implemented yet"); + public String[] listTagsForMetadataObject(String metalake, MetadataObject metadataObject) + throws NotFoundException { + return Arrays.stream(listTagsInfoForMetadataObject(metalake, metadataObject)) + .map(Tag::name) + .toArray(String[]::new); + } + + public Tag[] listTagsInfoForMetadataObject(String metalake, MetadataObject metadataObject) + throws NotFoundException { + NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + + return TreeLockUtils.doWithTreeLock( + entityIdent, + LockType.READ, + () -> { + try { + return supportsTagOperations + .listAssociatedTagsForMetadataObject(entityIdent, entityType) + .toArray(new Tag[0]); + } catch (NoSuchEntityException e) { + throw new NotFoundException( + e, "Failed to list tags for metadata object %s due to not found", metadataObject); + } catch (IOException e) { + LOG.error("Failed to list tags for metadata object {}", metadataObject, e); + throw new RuntimeException(e); + } + }); } public Tag getTagForMetadataObject(String metalake, MetadataObject metadataObject, String name) - throws NoSuchTagException { - throw new UnsupportedOperationException("Not implemented yet"); + throws NotFoundException { + NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + NameIdentifier tagIdent = ofTagIdent(metalake, name); + + return TreeLockUtils.doWithTreeLock( + entityIdent, + LockType.READ, + () -> { + try { + return supportsTagOperations.getTagForMetadataObject(entityIdent, entityType, tagIdent); + } catch (NoSuchEntityException e) { + if (e.getMessage().contains("No such tag entity")) { + throw new NoSuchTagException( + e, "Tag %s does not exist for metadata object %s", name, metadataObject); + } else { + throw new NotFoundException( + e, "Failed to get tag for metadata object %s due to not found", metadataObject); + } + } catch (IOException e) { + LOG.error("Failed to get tag for metadata object {}", metadataObject, e); + throw new RuntimeException(e); + } + }); } public String[] associateTagsForMetadataObject( - String metalake, MetadataObject metadataObject, String[] tagsToAdd, String[] tagsToRemove) { - throw new UnsupportedOperationException("Not implemented yet"); + String metalake, MetadataObject metadataObject, String[] tagsToAdd, String[] tagsToRemove) + throws NotFoundException, TagAlreadyAssociatedException { + Preconditions.checkArgument( + !metadataObject.type().equals(MetadataObject.Type.METALAKE) + && !metadataObject.type().equals(MetadataObject.Type.COLUMN), + "Cannot associate tags for unsupported metadata object type %s", + metadataObject.type()); + + NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + + // Remove all the tags that are both set to add and remove + Set tagsToAddSet = tagsToAdd == null ? Sets.newHashSet() : Sets.newHashSet(tagsToAdd); + Set tagsToRemoveSet = + tagsToRemove == null ? Sets.newHashSet() : Sets.newHashSet(tagsToRemove); + Set common = Sets.intersection(tagsToAddSet, tagsToRemoveSet).immutableCopy(); + tagsToAddSet.removeAll(common); + tagsToRemoveSet.removeAll(common); + + NameIdentifier[] tagsToAddIdent = + tagsToAddSet.stream().map(tag -> ofTagIdent(metalake, tag)).toArray(NameIdentifier[]::new); + NameIdentifier[] tagsToRemoveIdent = + tagsToRemoveSet.stream() + .map(tag -> ofTagIdent(metalake, tag)) + .toArray(NameIdentifier[]::new); + + // TODO. We need to add a write lock to Tag's namespace to avoid tag alteration and deletion + // during the association operation. + return TreeLockUtils.doWithTreeLock( + entityIdent, + LockType.READ, + () -> + TreeLockUtils.doWithTreeLock( + NameIdentifier.of(ofTagNamespace(metalake).levels()), + LockType.WRITE, + () -> { + try { + return supportsTagOperations + .associateTagsWithMetadataObject( + entityIdent, entityType, tagsToAddIdent, tagsToRemoveIdent) + .stream() + .map(Tag::name) + .toArray(String[]::new); + } catch (NoSuchEntityException e) { + throw new NotFoundException( + e, + "Failed to associate tags for metadata object %s due to not found", + metadataObject); + } catch (EntityAlreadyExistsException e) { + throw new TagAlreadyAssociatedException( + e, + "Failed to associate tags for metadata object due to some tags %s already " + + "associated to the metadata object %s", + Arrays.toString(tagsToAdd), + metadataObject); + } catch (IOException e) { + LOG.error("Failed to associate tags for metadata object {}", metadataObject, e); + throw new RuntimeException(e); + } + })); } private static void checkMetalakeExists(String metalake, EntityStore entityStore) { diff --git a/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java b/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java new file mode 100644 index 00000000000..5ede0a02095 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.utils; + +import com.google.common.base.Joiner; +import com.google.common.collect.BiMap; +import com.google.common.collect.ImmutableBiMap; +import org.apache.gravitino.Entity; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.NameIdentifier; + +import java.util.Optional; + +public class MetadataObjectUtil { + + private static final Joiner DOT = Joiner.on("."); + + private static final BiMap TYPE_TO_TYPE_MAP = + ImmutableBiMap.builder() + .put(MetadataObject.Type.METALAKE, Entity.EntityType.METALAKE) + .put(MetadataObject.Type.CATALOG, Entity.EntityType.CATALOG) + .put(MetadataObject.Type.SCHEMA, Entity.EntityType.SCHEMA) + .put(MetadataObject.Type.TABLE, Entity.EntityType.TABLE) + .put(MetadataObject.Type.TOPIC, Entity.EntityType.TOPIC) + .put(MetadataObject.Type.FILESET, Entity.EntityType.FILESET) + .put(MetadataObject.Type.COLUMN, Entity.EntityType.COLUMN) + .build(); + + private MetadataObjectUtil() {} + + /** + * Map the given {@link MetadataObject}'s type to the corresponding {@link Entity.EntityType}. + * + * @param metadataObject The metadata object + * @return The entity type + * @throws IllegalArgumentException if the metadata object type is unknown + */ + public static Entity.EntityType toEntityType(MetadataObject metadataObject) { + return Optional.ofNullable(TYPE_TO_TYPE_MAP.get(metadataObject.type())) + .orElseThrow( + () -> + new IllegalArgumentException( + "Unknown metadata object type: " + metadataObject.type())); + } + + /** + * Convert the given {@link MetadataObject} full name to the corresponding {@link NameIdentifier}. + * + * @param metalakeName The metalake name + * @param metadataObject The metadata object + * @return The entity identifier + * @throws IllegalArgumentException if the metadata object type is unsupported or unknown. + */ + public static NameIdentifier toEntityIdent(String metalakeName, MetadataObject metadataObject) { + switch (metadataObject.type()) { + case METALAKE: + return NameIdentifierUtil.ofMetalake(metalakeName); + case CATALOG: + case SCHEMA: + case TABLE: + case TOPIC: + case FILESET: + String fullName = DOT.join(metalakeName, metadataObject.fullName()); + return NameIdentifier.parse(fullName); + case COLUMN: + throw new IllegalArgumentException( + "Cannot convert column metadata object to entity identifier: " + + metadataObject.fullName()); + default: + throw new IllegalArgumentException( + "Unknown metadata object type: " + metadataObject.type()); + } + } +} diff --git a/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java b/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java index fcdaf43e0ca..616deb2359c 100644 --- a/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java +++ b/core/src/main/java/org/apache/gravitino/utils/NameIdentifierUtil.java @@ -18,8 +18,13 @@ */ package org.apache.gravitino.utils; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.errorprone.annotations.FormatMethod; import com.google.errorprone.annotations.FormatString; +import org.apache.gravitino.Entity; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.exceptions.IllegalNameIdentifierException; import org.apache.gravitino.exceptions.IllegalNamespaceException; @@ -193,4 +198,54 @@ public static void check(boolean expression, @FormatString String message, Objec throw new IllegalNamespaceException(message, args); } } + + /** + * Convert the given {@link NameIdentifier} and {@link Entity.EntityType} to {@link + * MetadataObject}. + * + * @param ident The identifier + * @param entityType The entity type + * @return The converted {@link MetadataObject} + */ + public static MetadataObject toMetadataObject( + NameIdentifier ident, Entity.EntityType entityType) { + Preconditions.checkArgument( + ident != null && entityType != null, "The identifier and entity type must not be null"); + + Joiner dot = Joiner.on("."); + + switch (entityType) { + case METALAKE: + checkMetalake(ident); + return MetadataObjects.of(null, ident.name(), MetadataObject.Type.METALAKE); + + case CATALOG: + checkCatalog(ident); + return MetadataObjects.of(null, ident.name(), MetadataObject.Type.CATALOG); + + case SCHEMA: + checkSchema(ident); + String schemaParent = ident.namespace().level(1); + return MetadataObjects.of(schemaParent, ident.name(), MetadataObject.Type.SCHEMA); + + case TABLE: + checkTable(ident); + String tableParent = dot.join(ident.namespace().level(1), ident.namespace().level(2)); + return MetadataObjects.of(tableParent, ident.name(), MetadataObject.Type.TABLE); + + case FILESET: + checkFileset(ident); + String filesetParent = dot.join(ident.namespace().level(1), ident.namespace().level(2)); + return MetadataObjects.of(filesetParent, ident.name(), MetadataObject.Type.FILESET); + + case TOPIC: + checkTopic(ident); + String topicParent = dot.join(ident.namespace().level(1), ident.namespace().level(2)); + return MetadataObjects.of(topicParent, ident.name(), MetadataObject.Type.TOPIC); + + default: + throw new IllegalArgumentException( + "Entity type " + entityType + " is not supported to convert to MetadataObject"); + } + } } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java index 06aec2335d1..6d6224792ca 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java @@ -23,9 +23,19 @@ import java.time.Instant; import java.util.List; import java.util.Map; + +import org.apache.gravitino.Entity; +import org.apache.gravitino.EntityAlreadyExistsException; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.BaseMetalake; +import org.apache.gravitino.meta.CatalogEntity; +import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.meta.TableEntity; import org.apache.gravitino.meta.TagEntity; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.TestJDBCBackend; @@ -297,4 +307,356 @@ public void testDeleteMetalake() throws IOException { NoSuchEntityException.class, () -> tagMetaService.getTagByIdentifier(TagManager.ofTagIdent(metalakeName + "1", "tag2"))); } + + @Test + public void testAssociateAndDisassociateTagsWithMetadataObject() throws IOException { + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog1", auditInfo); + backend.insert(catalog, false); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of(metalakeName, catalog.name()), + "schema1", + auditInfo); + backend.insert(schema, false); + + TableEntity table = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of(metalakeName, catalog.name(), schema.name()), + "table1", + auditInfo); + backend.insert(table, false); + + // Create tags to associate + TagMetaService tagMetaService = TagMetaService.getInstance(); + TagEntity tagEntity1 = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag1") + .withNamespace(TagManager.ofTagNamespace(metalakeName)) + .withComment("comment") + .withProperties(props) + .withAuditInfo(auditInfo) + .build(); + tagMetaService.insertTag(tagEntity1, false); + + TagEntity tagEntity2 = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag2") + .withNamespace(TagManager.ofTagNamespace(metalakeName)) + .withComment("comment") + .withProperties(props) + .withAuditInfo(auditInfo) + .build(); + tagMetaService.insertTag(tagEntity2, false); + + TagEntity tagEntity3 = + TagEntity.builder() + .withId(RandomIdGenerator.INSTANCE.nextId()) + .withName("tag3") + .withNamespace(TagManager.ofTagNamespace(metalakeName)) + .withComment("comment") + .withProperties(props) + .withAuditInfo(auditInfo) + .build(); + tagMetaService.insertTag(tagEntity3, false); + + // Test associate tags with metadata object + NameIdentifier[] tagsToAdd = + new NameIdentifier[] { + TagManager.ofTagIdent(metalakeName, "tag1"), + TagManager.ofTagIdent(metalakeName, "tag2"), + TagManager.ofTagIdent(metalakeName, "tag3") + }; + + List tagEntities = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToAdd, new NameIdentifier[0]); + Assertions.assertEquals(3, tagEntities.size()); + Assertions.assertTrue(tagEntities.contains(tagEntity1)); + Assertions.assertTrue(tagEntities.contains(tagEntity2)); + Assertions.assertTrue(tagEntities.contains(tagEntity3)); + + // Test disassociate tags with metadata object + NameIdentifier[] tagsToRemove = + new NameIdentifier[] {TagManager.ofTagIdent(metalakeName, "tag1")}; + + List tagEntities1 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), new NameIdentifier[0], tagsToRemove); + + Assertions.assertEquals(2, tagEntities1.size()); + Assertions.assertFalse(tagEntities1.contains(tagEntity1)); + Assertions.assertTrue(tagEntities1.contains(tagEntity2)); + Assertions.assertTrue(tagEntities1.contains(tagEntity3)); + + // Test no tags to associate and disassociate + List tagEntities2 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), new NameIdentifier[0], new NameIdentifier[0]); + Assertions.assertEquals(2, tagEntities2.size()); + Assertions.assertFalse(tagEntities2.contains(tagEntity1)); + Assertions.assertTrue(tagEntities2.contains(tagEntity2)); + Assertions.assertTrue(tagEntities2.contains(tagEntity3)); + + // Test associate and disassociate same tags with metadata object + List tagEntities3 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToRemove, tagsToRemove); + + Assertions.assertEquals(2, tagEntities3.size()); + Assertions.assertFalse(tagEntities3.contains(tagEntity1)); + Assertions.assertTrue(tagEntities3.contains(tagEntity2)); + Assertions.assertTrue(tagEntities3.contains(tagEntity3)); + + // Test associate and disassociate in-existent tags with metadata object + NameIdentifier[] tagsToAdd1 = + new NameIdentifier[] { + TagManager.ofTagIdent(metalakeName, "tag4"), TagManager.ofTagIdent(metalakeName, "tag5") + }; + + NameIdentifier[] tagsToRemove1 = + new NameIdentifier[] { + TagManager.ofTagIdent(metalakeName, "tag6"), TagManager.ofTagIdent(metalakeName, "tag7") + }; + + List tagEntities4 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToAdd1, tagsToRemove1); + + Assertions.assertEquals(2, tagEntities4.size()); + Assertions.assertTrue(tagEntities4.contains(tagEntity2)); + Assertions.assertTrue(tagEntities4.contains(tagEntity3)); + + // Test associate already associated tags with metadata object + Assertions.assertThrows( + EntityAlreadyExistsException.class, + () -> + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), tagsToAdd, new NameIdentifier[0])); + + // Test disassociate already disassociated tags with metadata object + List tagEntities5 = + tagMetaService.associateTagsWithMetadataObject( + catalog.nameIdentifier(), catalog.type(), new NameIdentifier[0], tagsToRemove); + + Assertions.assertEquals(2, tagEntities5.size()); + Assertions.assertTrue(tagEntities5.contains(tagEntity2)); + Assertions.assertTrue(tagEntities5.contains(tagEntity3)); + + // Test associate and disassociate with invalid metadata object + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.associateTagsWithMetadataObject( + NameIdentifier.of(metalakeName, "non-existent-catalog"), + catalog.type(), + tagsToAdd, + tagsToRemove)); + + // Test associate and disassociate to a schema + List tagEntities6 = + tagMetaService.associateTagsWithMetadataObject( + schema.nameIdentifier(), schema.type(), tagsToAdd, tagsToRemove); + + Assertions.assertEquals(2, tagEntities6.size()); + Assertions.assertTrue(tagEntities6.contains(tagEntity2)); + Assertions.assertTrue(tagEntities6.contains(tagEntity3)); + + // Test associate and disassociate to a table + List tagEntities7 = + tagMetaService.associateTagsWithMetadataObject( + table.nameIdentifier(), table.type(), tagsToAdd, tagsToRemove); + + Assertions.assertEquals(2, tagEntities7.size()); + Assertions.assertTrue(tagEntities7.contains(tagEntity2)); + Assertions.assertTrue(tagEntities7.contains(tagEntity3)); + } + + @Test + public void testListTagsForMetadataObject() throws IOException { + testAssociateAndDisassociateTagsWithMetadataObject(); + + TagMetaService tagMetaService = TagMetaService.getInstance(); + + // Test list tags for catalog + List tagEntities = + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1"), Entity.EntityType.CATALOG); + Assertions.assertEquals(2, tagEntities.size()); + Assertions.assertTrue( + tagEntities.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag2"))); + Assertions.assertTrue( + tagEntities.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag3"))); + + // Test list tags for schema + List tagEntities1 = + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1"), Entity.EntityType.SCHEMA); + + Assertions.assertEquals(2, tagEntities1.size()); + Assertions.assertTrue( + tagEntities1.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag2"))); + Assertions.assertTrue( + tagEntities1.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag3"))); + + // Test list tags for table + List tagEntities2 = + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE); + + Assertions.assertEquals(2, tagEntities2.size()); + Assertions.assertTrue( + tagEntities2.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag2"))); + Assertions.assertTrue( + tagEntities2.stream().anyMatch(tagEntity -> tagEntity.name().equals("tag3"))); + + // Test list tags for non-existent metadata object + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.listTagsForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table2"), + Entity.EntityType.TABLE)); + } + + @Test + public void testGetTagForMetadataObject() throws IOException { + testAssociateAndDisassociateTagsWithMetadataObject(); + + TagMetaService tagMetaService = TagMetaService.getInstance(); + + // Test get tag for catalog + TagEntity tagEntity = + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1"), + Entity.EntityType.CATALOG, + TagManager.ofTagIdent(metalakeName, "tag2")); + Assertions.assertEquals("tag2", tagEntity.name()); + + // Test get tag for schema + TagEntity tagEntity1 = + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1"), + Entity.EntityType.SCHEMA, + TagManager.ofTagIdent(metalakeName, "tag3")); + Assertions.assertEquals("tag3", tagEntity1.name()); + + // Test get tag for table + TagEntity tagEntity2 = + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE, + TagManager.ofTagIdent(metalakeName, "tag2")); + Assertions.assertEquals("tag2", tagEntity2.name()); + + // Test get tag for non-existent metadata object + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table2"), + Entity.EntityType.TABLE, + TagManager.ofTagIdent(metalakeName, "tag2"))); + + // Test get tag for non-existent tag + Throwable e = + Assertions.assertThrows( + NoSuchEntityException.class, + () -> + tagMetaService.getTagForMetadataObject( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE, + TagManager.ofTagIdent(metalakeName, "tag4"))); + Assertions.assertTrue(e.getMessage().contains("No such tag entity: tag4")); + } + + @Test + public void testListAssociatedMetadataObjectsForTag() throws IOException { + testAssociateAndDisassociateTagsWithMetadataObject(); + + TagMetaService tagMetaService = TagMetaService.getInstance(); + + // Test list associated metadata objects for tag2 + List metadataObjects = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(3, metadataObjects.size()); + Assertions.assertTrue( + metadataObjects.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + Assertions.assertTrue( + metadataObjects.contains( + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA))); + Assertions.assertTrue( + metadataObjects.contains( + MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE))); + + // Test list associated metadata objects for tag3 + List metadataObjects1 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag3")); + + Assertions.assertEquals(3, metadataObjects1.size()); + Assertions.assertTrue( + metadataObjects1.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + Assertions.assertTrue( + metadataObjects1.contains( + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA))); + Assertions.assertTrue( + metadataObjects1.contains( + MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE))); + + // Test list associated metadata objects for non-existent tag + List metadataObjects2 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag4")); + Assertions.assertEquals(0, metadataObjects2.size()); + + // Test metadata object non-exist scenario. + backend.delete( + NameIdentifier.of(metalakeName, "catalog1", "schema1", "table1"), + Entity.EntityType.TABLE, + false); + + List metadataObjects3 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(2, metadataObjects3.size()); + Assertions.assertTrue( + metadataObjects3.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + Assertions.assertTrue( + metadataObjects3.contains( + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA))); + + backend.delete( + NameIdentifier.of(metalakeName, "catalog1", "schema1"), Entity.EntityType.SCHEMA, false); + + List metadataObjects4 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(1, metadataObjects4.size()); + Assertions.assertTrue( + metadataObjects4.contains(MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG))); + + backend.delete(NameIdentifier.of(metalakeName, "catalog1"), Entity.EntityType.CATALOG, false); + + List metadataObjects5 = + tagMetaService.listAssociatedMetadataObjectIdentsForTag( + TagManager.ofTagIdent(metalakeName, "tag2")); + + Assertions.assertEquals(0, metadataObjects5.size()); + } } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java index a7b5e08553e..ccf445c815f 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java @@ -34,6 +34,7 @@ import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.Namespace; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.json.JsonUtils; @@ -52,6 +53,7 @@ import org.apache.gravitino.storage.relational.po.MetalakePO; import org.apache.gravitino.storage.relational.po.SchemaPO; import org.apache.gravitino.storage.relational.po.TablePO; +import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.po.TagPO; import org.apache.gravitino.storage.relational.po.TopicPO; import org.apache.gravitino.utils.NamespaceUtil; @@ -654,6 +656,21 @@ public void testUpdateTagPOVersion() { assertEquals("this is test2", updatePO.getComment()); } + @Test + public void testTagMetadataObjectRelPO() { + TagMetadataObjectRelPO tagMetadataObjectRelPO = + POConverters.initializeTagMetadataObjectRelPOWithVersion( + 1L, 1L, MetadataObject.Type.CATALOG.toString()); + assertEquals(1L, tagMetadataObjectRelPO.getTagId()); + assertEquals(1L, tagMetadataObjectRelPO.getMetadataObjectId()); + assertEquals( + MetadataObject.Type.CATALOG.toString(), tagMetadataObjectRelPO.getMetadataObjectType()); + + assertEquals(1, tagMetadataObjectRelPO.getCurrentVersion()); + assertEquals(1, tagMetadataObjectRelPO.getLastVersion()); + assertEquals(0, tagMetadataObjectRelPO.getDeletedAt()); + } + private static BaseMetalake createMetalake(Long id, String name, String comment) { AuditInfo auditInfo = AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); diff --git a/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java b/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java index efd78a77a17..5154ef7bb94 100644 --- a/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java +++ b/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java @@ -43,19 +43,29 @@ import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.gravitino.Catalog; import org.apache.gravitino.Config; +import org.apache.gravitino.Entity; import org.apache.gravitino.EntityStore; import org.apache.gravitino.EntityStoreFactory; import org.apache.gravitino.GravitinoEnv; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchTagException; +import org.apache.gravitino.exceptions.NotFoundException; +import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; import org.apache.gravitino.exceptions.TagAlreadyExistsException; import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.BaseMetalake; +import org.apache.gravitino.meta.CatalogEntity; +import org.apache.gravitino.meta.SchemaEntity; import org.apache.gravitino.meta.SchemaVersion; +import org.apache.gravitino.meta.TableEntity; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; +import org.apache.gravitino.utils.NameIdentifierUtil; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -73,6 +83,12 @@ public class TestTagManager { private static final String METALAKE = "metalake_for_tag_test"; + private static final String CATALOG = "catalog_for_tag_test"; + + private static final String SCHEMA = "schema_for_tag_test"; + + private static final String TABLE = "table_for_tag_test"; + private static EntityStore entityStore; private static IdGenerator idGenerator; @@ -115,6 +131,37 @@ public static void setUp() throws IOException, IllegalAccessException { .build(); entityStore.put(metalake, false /* overwritten */); + CatalogEntity catalog = + CatalogEntity.builder() + .withId(idGenerator.nextId()) + .withName(CATALOG) + .withNamespace(Namespace.of(METALAKE)) + .withType(Catalog.Type.RELATIONAL) + .withProvider("test") + .withComment("Test catalog") + .withAuditInfo(audit) + .build(); + entityStore.put(catalog, false /* overwritten */); + + SchemaEntity schema = + SchemaEntity.builder() + .withId(idGenerator.nextId()) + .withName(SCHEMA) + .withNamespace(Namespace.of(METALAKE, CATALOG)) + .withComment("Test schema") + .withAuditInfo(audit) + .build(); + entityStore.put(schema, false /* overwritten */); + + TableEntity table = + TableEntity.builder() + .withId(idGenerator.nextId()) + .withName(TABLE) + .withNamespace(Namespace.of(METALAKE, CATALOG, SCHEMA)) + .withAuditInfo(audit) + .build(); + entityStore.put(table, false /* overwritten */); + tagManager = new TagManager(idGenerator, entityStore); } @@ -130,6 +177,24 @@ public static void tearDown() throws IOException { @AfterEach public void cleanUp() { + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + String[] catalogTags = tagManager.listTagsForMetadataObject(METALAKE, catalogObject); + tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, null, catalogTags); + + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + String[] schemaTags = tagManager.listTagsForMetadataObject(METALAKE, schemaObject); + tagManager.associateTagsForMetadataObject(METALAKE, schemaObject, null, schemaTags); + + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + String[] tableTags = tagManager.listTagsForMetadataObject(METALAKE, tableObject); + tagManager.associateTagsForMetadataObject(METALAKE, tableObject, null, tableTags); + Arrays.stream(tagManager.listTags(METALAKE)).forEach(n -> tagManager.deleteTag(METALAKE, n)); } @@ -245,4 +310,285 @@ public void testDeleteTag() { () -> tagManager.deleteTag("non_existent_metalake", "tag1")); Assertions.assertEquals("Metalake non_existent_metalake does not exist", e.getMessage()); } + + @Test + public void testAssociateTagsForMetadataObject() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + // Test associate tags for catalog + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + String[] tagsToAdd = new String[] {tag1.name(), tag2.name(), tag3.name()}; + + String[] tags = + tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, tagsToAdd, null); + + Assertions.assertEquals(3, tags.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags)); + + // Test disassociate tags for catalog + String[] tagsToRemove = new String[] {tag1.name()}; + String[] tags1 = + tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, null, tagsToRemove); + + Assertions.assertEquals(2, tags1.length); + Assertions.assertEquals(ImmutableSet.of("tag2", "tag3"), ImmutableSet.copyOf(tags1)); + + // Test associate and disassociate no tags for catalog + String[] tags2 = tagManager.associateTagsForMetadataObject(METALAKE, catalogObject, null, null); + + Assertions.assertEquals(2, tags2.length); + Assertions.assertEquals(ImmutableSet.of("tag2", "tag3"), ImmutableSet.copyOf(tags2)); + + // Test re-associate tags for catalog + Throwable e = + Assertions.assertThrows( + TagAlreadyAssociatedException.class, + () -> + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, tagsToAdd, null)); + Assertions.assertTrue(e.getMessage().contains("Failed to associate tags for metadata object")); + + // Test associate and disassociate non-existent tags for catalog + String[] tags3 = + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {"tag4", "tag5"}, new String[] {"tag6"}); + + Assertions.assertEquals(2, tags3.length); + Assertions.assertEquals(ImmutableSet.of("tag2", "tag3"), ImmutableSet.copyOf(tags3)); + + // Test associate tags for non-existent metadata object + MetadataObject nonExistentObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, "non_existent_catalog"), + Entity.EntityType.CATALOG); + Throwable e1 = + Assertions.assertThrows( + NotFoundException.class, + () -> + tagManager.associateTagsForMetadataObject( + METALAKE, nonExistentObject, tagsToAdd, null)); + Assertions.assertTrue(e1.getMessage().contains("Failed to associate tags for metadata object")); + + // Test associate tags for unsupported metadata object + MetadataObject metalakeObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofMetalake(METALAKE), Entity.EntityType.METALAKE); + Throwable e2 = + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + tagManager.associateTagsForMetadataObject( + METALAKE, metalakeObject, tagsToAdd, null)); + Assertions.assertTrue( + e2.getMessage().contains("Cannot associate tags for unsupported metadata object type")); + + // Test associate tags for schema + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + String[] tags4 = + tagManager.associateTagsForMetadataObject(METALAKE, schemaObject, tagsToAdd, null); + + Assertions.assertEquals(3, tags4.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags4)); + + // Test associate tags for table + String[] tagsToAdd1 = new String[] {tag1.name()}; + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + String[] tags5 = + tagManager.associateTagsForMetadataObject(METALAKE, tableObject, tagsToAdd1, null); + + Assertions.assertEquals(1, tags5.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tags5)); + + // Test associate and disassociate same tags for table + String[] tagsToAdd2 = new String[] {tag2.name(), tag3.name()}; + String[] tagsToRemove1 = new String[] {tag2.name()}; + String[] tags6 = + tagManager.associateTagsForMetadataObject(METALAKE, tableObject, tagsToAdd2, tagsToRemove1); + + Assertions.assertEquals(2, tags6.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag3"), ImmutableSet.copyOf(tags6)); + } + + @Test + public void testListMetadataObjectsForTag() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, schemaObject, new String[] {tag1.name(), tag2.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, tableObject, new String[] {tag1.name()}, null); + + MetadataObject[] objects = tagManager.listMetadataObjectsForTag(METALAKE, tag1.name()); + Assertions.assertEquals(3, objects.length); + Assertions.assertEquals( + ImmutableSet.of(catalogObject, schemaObject, tableObject), ImmutableSet.copyOf(objects)); + + MetadataObject[] objects1 = tagManager.listMetadataObjectsForTag(METALAKE, tag2.name()); + Assertions.assertEquals(2, objects1.length); + Assertions.assertEquals( + ImmutableSet.of(catalogObject, schemaObject), ImmutableSet.copyOf(objects1)); + + MetadataObject[] objects2 = tagManager.listMetadataObjectsForTag(METALAKE, tag3.name()); + Assertions.assertEquals(1, objects2.length); + Assertions.assertEquals(ImmutableSet.of(catalogObject), ImmutableSet.copyOf(objects2)); + + // List metadata objects for non-existent tag + Throwable e = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.listMetadataObjectsForTag(METALAKE, "non_existent_tag")); + Assertions.assertTrue( + e.getMessage() + .contains( + "Tag with name non_existent_tag under metalake " + METALAKE + " does not exist")); + } + + @Test + public void testListTagsForMetadataObject() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, schemaObject, new String[] {tag1.name(), tag2.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, tableObject, new String[] {tag1.name()}, null); + + String[] tags = tagManager.listTagsForMetadataObject(METALAKE, catalogObject); + Assertions.assertEquals(3, tags.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2", "tag3"), ImmutableSet.copyOf(tags)); + + Tag[] tagsInfo = tagManager.listTagsInfoForMetadataObject(METALAKE, catalogObject); + Assertions.assertEquals(3, tagsInfo.length); + Assertions.assertEquals(ImmutableSet.of(tag1, tag2, tag3), ImmutableSet.copyOf(tagsInfo)); + + String[] tags1 = tagManager.listTagsForMetadataObject(METALAKE, schemaObject); + Assertions.assertEquals(2, tags1.length); + Assertions.assertEquals(ImmutableSet.of("tag1", "tag2"), ImmutableSet.copyOf(tags1)); + + Tag[] tagsInfo1 = tagManager.listTagsInfoForMetadataObject(METALAKE, schemaObject); + Assertions.assertEquals(2, tagsInfo1.length); + Assertions.assertEquals(ImmutableSet.of(tag1, tag2), ImmutableSet.copyOf(tagsInfo1)); + + String[] tags2 = tagManager.listTagsForMetadataObject(METALAKE, tableObject); + Assertions.assertEquals(1, tags2.length); + Assertions.assertEquals(ImmutableSet.of("tag1"), ImmutableSet.copyOf(tags2)); + + Tag[] tagsInfo2 = tagManager.listTagsInfoForMetadataObject(METALAKE, tableObject); + Assertions.assertEquals(1, tagsInfo2.length); + Assertions.assertEquals(ImmutableSet.of(tag1), ImmutableSet.copyOf(tagsInfo2)); + + // List tags for non-existent metadata object + MetadataObject nonExistentObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, "non_existent_catalog"), + Entity.EntityType.CATALOG); + Throwable e = + Assertions.assertThrows( + NotFoundException.class, + () -> tagManager.listTagsForMetadataObject(METALAKE, nonExistentObject)); + Assertions.assertTrue( + e.getMessage().contains("Failed to list tags for metadata object " + nonExistentObject)); + } + + @Test + public void testGetTagForMetadataObject() { + Tag tag1 = tagManager.createTag(METALAKE, "tag1", null, null); + Tag tag2 = tagManager.createTag(METALAKE, "tag2", null, null); + Tag tag3 = tagManager.createTag(METALAKE, "tag3", null, null); + + MetadataObject catalogObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, CATALOG), Entity.EntityType.CATALOG); + MetadataObject schemaObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofSchema(METALAKE, CATALOG, SCHEMA), Entity.EntityType.SCHEMA); + MetadataObject tableObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofTable(METALAKE, CATALOG, SCHEMA, TABLE), Entity.EntityType.TABLE); + + tagManager.associateTagsForMetadataObject( + METALAKE, catalogObject, new String[] {tag1.name(), tag2.name(), tag3.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, schemaObject, new String[] {tag1.name(), tag2.name()}, null); + tagManager.associateTagsForMetadataObject( + METALAKE, tableObject, new String[] {tag1.name()}, null); + + Tag result = tagManager.getTagForMetadataObject(METALAKE, catalogObject, tag1.name()); + Assertions.assertEquals(tag1, result); + + Tag result1 = tagManager.getTagForMetadataObject(METALAKE, schemaObject, tag1.name()); + Assertions.assertEquals(tag1, result1); + + Tag result2 = tagManager.getTagForMetadataObject(METALAKE, schemaObject, tag2.name()); + Assertions.assertEquals(tag2, result2); + + Tag result3 = tagManager.getTagForMetadataObject(METALAKE, catalogObject, tag3.name()); + Assertions.assertEquals(tag3, result3); + + // Test get non-existent tag for metadata object + Throwable e = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, catalogObject, "non_existent_tag")); + Assertions.assertTrue(e.getMessage().contains("Tag non_existent_tag does not exist")); + + Throwable e1 = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, schemaObject, tag3.name())); + Assertions.assertTrue(e1.getMessage().contains("Tag tag3 does not exist")); + + Throwable e2 = + Assertions.assertThrows( + NoSuchTagException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, tableObject, tag2.name())); + Assertions.assertTrue(e2.getMessage().contains("Tag tag2 does not exist")); + + // Test get tag for non-existent metadata object + MetadataObject nonExistentObject = + NameIdentifierUtil.toMetadataObject( + NameIdentifierUtil.ofCatalog(METALAKE, "non_existent_catalog"), + Entity.EntityType.CATALOG); + Throwable e3 = + Assertions.assertThrows( + NotFoundException.class, + () -> tagManager.getTagForMetadataObject(METALAKE, nonExistentObject, tag1.name())); + Assertions.assertTrue( + e3.getMessage().contains("Failed to get tag for metadata object " + nonExistentObject)); + } } diff --git a/core/src/test/java/org/apache/gravitino/utils/TestNameIdentifierUtil.java b/core/src/test/java/org/apache/gravitino/utils/TestNameIdentifierUtil.java index 487c0084d0c..964f910ba39 100644 --- a/core/src/test/java/org/apache/gravitino/utils/TestNameIdentifierUtil.java +++ b/core/src/test/java/org/apache/gravitino/utils/TestNameIdentifierUtil.java @@ -18,9 +18,13 @@ */ package org.apache.gravitino.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.gravitino.Entity; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.exceptions.IllegalNameIdentifierException; import org.apache.gravitino.exceptions.IllegalNamespaceException; @@ -58,4 +62,66 @@ public void testCheckNameIdentifier() { assertThrows(IllegalNamespaceException.class, () -> NameIdentifierUtil.checkTable(abc)); assertTrue(excep3.getMessage().contains("Table namespace must be non-null and have 3 levels")); } + + @Test + public void testToMetadataObject() { + // test metalake + NameIdentifier metalake = NameIdentifier.of("metalake1"); + MetadataObject metalakeObject = + MetadataObjects.parse("metalake1", MetadataObject.Type.METALAKE); + assertEquals( + metalakeObject, NameIdentifierUtil.toMetadataObject(metalake, Entity.EntityType.METALAKE)); + + // test catalog + NameIdentifier catalog = NameIdentifier.of("metalake1", "catalog1"); + MetadataObject catalogObject = MetadataObjects.parse("catalog1", MetadataObject.Type.CATALOG); + assertEquals( + catalogObject, NameIdentifierUtil.toMetadataObject(catalog, Entity.EntityType.CATALOG)); + + // test schema + NameIdentifier schema = NameIdentifier.of("metalake1", "catalog1", "schema1"); + MetadataObject schemaObject = + MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA); + assertEquals( + schemaObject, NameIdentifierUtil.toMetadataObject(schema, Entity.EntityType.SCHEMA)); + + // test table + NameIdentifier table = NameIdentifier.of("metalake1", "catalog1", "schema1", "table1"); + MetadataObject tableObject = + MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE); + assertEquals(tableObject, NameIdentifierUtil.toMetadataObject(table, Entity.EntityType.TABLE)); + + // test topic + NameIdentifier topic = NameIdentifier.of("metalake1", "catalog1", "schema1", "topic1"); + MetadataObject topicObject = + MetadataObjects.parse("catalog1.schema1.topic1", MetadataObject.Type.TOPIC); + assertEquals(topicObject, NameIdentifierUtil.toMetadataObject(topic, Entity.EntityType.TOPIC)); + + // test fileset + NameIdentifier fileset = NameIdentifier.of("metalake1", "catalog1", "schema1", "fileset1"); + MetadataObject filesetObject = + MetadataObjects.parse("catalog1.schema1.fileset1", MetadataObject.Type.FILESET); + assertEquals( + filesetObject, NameIdentifierUtil.toMetadataObject(fileset, Entity.EntityType.FILESET)); + + // test column + Throwable e = + assertThrows( + IllegalArgumentException.class, + () -> NameIdentifierUtil.toMetadataObject(fileset, Entity.EntityType.COLUMN)); + assertTrue(e.getMessage().contains("Entity type COLUMN is not supported")); + + // test null + Throwable e1 = + assertThrows( + IllegalArgumentException.class, () -> NameIdentifierUtil.toMetadataObject(null, null)); + assertTrue(e1.getMessage().contains("The identifier and entity type must not be null")); + + // test unsupported + Throwable e2 = + assertThrows( + IllegalArgumentException.class, + () -> NameIdentifierUtil.toMetadataObject(fileset, Entity.EntityType.TAG)); + assertTrue(e2.getMessage().contains("Entity type TAG is not supported")); + } } diff --git a/scripts/h2/schema-h2.sql b/scripts/h2/schema-h2.sql index f4c961cb61d..0ebf91fa39a 100644 --- a/scripts/h2/schema-h2.sql +++ b/scripts/h2/schema-h2.sql @@ -241,7 +241,7 @@ CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', PRIMARY KEY (`id`), - UNIQUE KEY `uk_ti_mi_del` (`tag_id`, `metadata_object_id`, `deleted_at`), + UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), KEY `idx_tid` (`tag_id`), KEY `idx_mid` (`metadata_object_id`) ) ENGINE=InnoDB; diff --git a/scripts/mysql/schema-0.6.0-mysql.sql b/scripts/mysql/schema-0.6.0-mysql.sql index 8418a5602aa..b50609da3c4 100644 --- a/scripts/mysql/schema-0.6.0-mysql.sql +++ b/scripts/mysql/schema-0.6.0-mysql.sql @@ -234,7 +234,7 @@ CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', PRIMARY KEY (`id`), - UNIQUE KEY `uk_ti_mi_del` (`tag_id`, `metadata_object_id`, `deleted_at`), + UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), KEY `idx_tid` (`tag_id`), KEY `idx_mid` (`metadata_object_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata object relation'; diff --git a/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql b/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql index 8fb71f73012..3048c67abe3 100644 --- a/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql +++ b/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql @@ -60,7 +60,7 @@ CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', PRIMARY KEY (`id`), - UNIQUE KEY `uk_ti_mi_del` (`tag_id`, `metadata_object_id`, `deleted_at`), + UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), KEY `idx_tid` (`tag_id`), KEY `idx_mid` (`metadata_object_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata object relation'; From 971b9c9cacbb0cf8219383a1b4dcc44af72a4a76 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 16 Jul 2024 11:52:26 +0800 Subject: [PATCH 22/27] Fix style issues --- .../relational/mapper/TagMetadataObjectRelMapper.java | 1 - .../storage/relational/service/MetadataObjectService.java | 6 ------ .../storage/relational/service/RoleMetaService.java | 3 ++- .../org/apache/gravitino/tag/SupportsTagOperations.java | 6 ++---- core/src/main/java/org/apache/gravitino/tag/TagManager.java | 1 - .../java/org/apache/gravitino/utils/MetadataObjectUtil.java | 3 +-- .../storage/relational/service/TestTagMetaService.java | 1 - 7 files changed, 5 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index 06fa6e745cd..06acec7f3dd 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -19,7 +19,6 @@ package org.apache.gravitino.storage.relational.mapper; import java.util.List; - import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.po.TagPO; import org.apache.ibatis.annotations.Delete; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index b996c52dc11..1fa5de878ac 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -31,12 +31,6 @@ import org.apache.gravitino.storage.relational.po.SchemaPO; import org.apache.gravitino.storage.relational.po.TablePO; import org.apache.gravitino.storage.relational.po.TopicPO; -import org.apache.gravitino.storage.relational.service.CatalogMetaService; -import org.apache.gravitino.storage.relational.service.FilesetMetaService; -import org.apache.gravitino.storage.relational.service.MetalakeMetaService; -import org.apache.gravitino.storage.relational.service.SchemaMetaService; -import org.apache.gravitino.storage.relational.service.TableMetaService; -import org.apache.gravitino.storage.relational.service.TopicMetaService; /** * MetadataObjectService is used for converting full name to entity id and converting entity id to diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index f4c2cb7d7c5..cf8a5632a2f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -107,7 +107,8 @@ public void insertRole(RoleEntity roleEntity, boolean overwritten) throws IOExce POConverters.initializeSecurablePOBuilderWithVersion( roleEntity.id(), object, getEntityType(object)); objectBuilder.withEntityId( - MetadataObjectService.getMetadataObjectId(metalakeId, object.fullName(), object.type())); + MetadataObjectService.getMetadataObjectId( + metalakeId, object.fullName(), object.type())); securableObjectPOs.add(objectBuilder.build()); } diff --git a/core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java b/core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java index 07157d58809..fd635a02412 100644 --- a/core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java +++ b/core/src/main/java/org/apache/gravitino/tag/SupportsTagOperations.java @@ -19,7 +19,8 @@ package org.apache.gravitino.tag; - +import java.io.IOException; +import java.util.List; import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; @@ -28,9 +29,6 @@ import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.meta.TagEntity; -import java.io.IOException; -import java.util.List; - /** * An interface to support extra tag operations, this interface should be mixed with {@link * EntityStore} to provide extra operations. diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index 99e3d4fc81b..061472da79e 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.Map; import java.util.Set; - import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; diff --git a/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java b/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java index 5ede0a02095..587e1eaa4f7 100644 --- a/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java +++ b/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java @@ -21,12 +21,11 @@ import com.google.common.base.Joiner; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; +import java.util.Optional; import org.apache.gravitino.Entity; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; -import java.util.Optional; - public class MetadataObjectUtil { private static final Joiner DOT = Joiner.on("."); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java index 6d6224792ca..185b6a2c199 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java @@ -23,7 +23,6 @@ import java.time.Instant; import java.util.List; import java.util.Map; - import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.MetadataObject; From f1d9cca25df33702091fe28916a78e3fb1c719fc Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 16 Jul 2024 11:56:09 +0800 Subject: [PATCH 23/27] Address the comments --- core/src/main/java/org/apache/gravitino/EntityStore.java | 2 +- .../gravitino/storage/relational/mapper/TagMetaMapper.java | 2 +- .../storage/relational/mapper/TagMetadataObjectRelMapper.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/EntityStore.java b/core/src/main/java/org/apache/gravitino/EntityStore.java index 7831e510e3d..3693c5bbf23 100644 --- a/core/src/main/java/org/apache/gravitino/EntityStore.java +++ b/core/src/main/java/org/apache/gravitino/EntityStore.java @@ -192,6 +192,6 @@ R executeInTransaction(Executable executable) * @throws UnsupportedOperationException if the extra operations are not supported */ default SupportsTagOperations tagOperations() { - throw new UnsupportedOperationException("extra operations are not supported"); + throw new UnsupportedOperationException("tag operations are not supported"); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java index 273a8771c5f..7b52d34a147 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetaMapper.java @@ -62,7 +62,7 @@ public interface TagMetaMapper { + " tm JOIN " + MetalakeMetaMapper.TABLE_NAME + " mm ON tm.metalake_id = mm.metalake_id" - + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name IN" + + " WHERE mm.metalake_name = #{metalakeName} AND tm.tag_name IN " + " " + " #{tagName}" diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index 06acec7f3dd..1ade27d8ffc 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -112,7 +112,7 @@ List listTagMetadataObjectRelsByMetalakeAndTagName( + TAG_METADATA_OBJECT_RELATION_TABLE_NAME + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)" + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000" - + " WHERE tag_id IN", + + " WHERE tag_id IN ", "", "#{tagId}", "", From 1a8bbdb9ceeef3e3bf30de6ea99a54df8611dec3 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 16 Jul 2024 15:12:01 +0800 Subject: [PATCH 24/27] Fix style issues --- .../dto/requests/TagUpdateRequest.java | 4 +- .../responses/MetadataObjectListResponse.java | 10 ++--- .../dto/responses/TagListResponse.java | 2 +- .../gravitino/dto/responses/TagResponse.java | 2 +- .../dto/requests/TestTagUpdatesRequest.java | 1 - .../relational/RelationalEntityStore.java | 3 +- .../mapper/TagMetadataObjectRelMapper.java | 1 - .../org/apache/gravitino/tag/TagManager.java | 1 - .../server/web/rest/TagOperations.java | 38 +++++++++---------- .../server/web/rest/TestTagOperations.java | 4 +- 10 files changed, 32 insertions(+), 34 deletions(-) diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/TagUpdateRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/TagUpdateRequest.java index ce3bec897fd..5323ac56548 100644 --- a/common/src/main/java/org/apache/gravitino/dto/requests/TagUpdateRequest.java +++ b/common/src/main/java/org/apache/gravitino/dto/requests/TagUpdateRequest.java @@ -18,8 +18,6 @@ */ package org.apache.gravitino.dto.requests; -import org.apache.gravitino.rest.RESTRequest; -import org.apache.gravitino.tag.TagChange; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; @@ -29,6 +27,8 @@ import lombok.Getter; import lombok.ToString; import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.rest.RESTRequest; +import org.apache.gravitino.tag.TagChange; /** Represents a request to update a tag. */ @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java index 40eddece55e..91a1f61eff2 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java @@ -18,15 +18,14 @@ */ package org.apache.gravitino.dto.responses; -import org.apache.gravitino.dto.tag.MetadataObjectDTO; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import java.util.Arrays; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; +import org.apache.gravitino.dto.tag.MetadataObjectDTO; /** Represents a response containing a list of metadata objects. */ @Getter @@ -65,7 +64,8 @@ public void validate() throws IllegalArgumentException { Arrays.stream(metadataObjects) .forEach( object -> - Preconditions.checkArgument(object != null && StringUtils.isNoneBlank(object.name()), - "metadataObject must not be null and empty")); + Preconditions.checkArgument( + object != null && StringUtils.isNoneBlank(object.name()), + "metadataObject must not be null and empty")); } } diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java index cc90a57306f..428f3998680 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java @@ -18,12 +18,12 @@ */ package org.apache.gravitino.dto.responses; -import org.apache.gravitino.dto.tag.TagDTO; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.apache.gravitino.dto.tag.TagDTO; /** Represents a response for a list of tags. */ @Getter diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/TagResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/TagResponse.java index 90428d54517..9f154e4ed5b 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/TagResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/TagResponse.java @@ -18,12 +18,12 @@ */ package org.apache.gravitino.dto.responses; -import org.apache.gravitino.dto.tag.TagDTO; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.apache.gravitino.dto.tag.TagDTO; /** Represents a response for a tag. */ @Getter diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestTagUpdatesRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestTagUpdatesRequest.java index cac5e2c63b7..8fb0a0bae45 100644 --- a/common/src/test/java/org/apache/gravitino/dto/requests/TestTagUpdatesRequest.java +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestTagUpdatesRequest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableList; import java.util.List; - import org.apache.gravitino.json.JsonUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java index 6c8290e6af2..9d5c0b21cd3 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/RelationalEntityStore.java @@ -20,11 +20,10 @@ import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.List; import java.util.function.Function; - -import com.google.common.collect.ImmutableMap; import org.apache.gravitino.Config; import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java index 85f2d2609a8..1ade27d8ffc 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java @@ -19,7 +19,6 @@ package org.apache.gravitino.storage.relational.mapper; import java.util.List; - import org.apache.gravitino.storage.relational.po.TagMetadataObjectRelPO; import org.apache.gravitino.storage.relational.po.TagPO; import org.apache.ibatis.annotations.Delete; diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index 0b25b837774..1726350a561 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.Map; import java.util.Set; - import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java index 30f3884482f..b27ee7dcf46 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java @@ -20,6 +20,25 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import org.apache.commons.lang3.ArrayUtils; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.dto.requests.TagCreateRequest; @@ -40,25 +59,6 @@ import org.apache.gravitino.tag.Tag; import org.apache.gravitino.tag.TagChange; import org.apache.gravitino.tag.TagManager; -import com.google.common.collect.Lists; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Optional; -import javax.inject.Inject; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.Response; -import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java index de0da32f637..8dc2eb22d91 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java @@ -38,10 +38,11 @@ import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; - import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.dto.requests.TagCreateRequest; +import org.apache.gravitino.dto.requests.TagUpdateRequest; +import org.apache.gravitino.dto.requests.TagUpdatesRequest; import org.apache.gravitino.dto.requests.TagsAssociateRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.ErrorConstants; @@ -56,6 +57,7 @@ import org.apache.gravitino.exceptions.TagAlreadyExistsException; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.TagEntity; +import org.apache.gravitino.rest.RESTUtils; import org.apache.gravitino.tag.Tag; import org.apache.gravitino.tag.TagChange; import org.apache.gravitino.tag.TagManager; From cb4ebe79a8704df39c9ba4ad9cbf84e539a0dbe4 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 16 Jul 2024 17:01:26 +0800 Subject: [PATCH 25/27] Improve the code --- .../storage/relational/JDBCBackend.java | 2 +- .../relational/service/TagMetaService.java | 2 +- .../org/apache/gravitino/tag/TagManager.java | 50 ++++++- .../gravitino/utils/MetadataObjectUtil.java | 8 ++ .../service/TestTagMetaService.java | 12 +- .../apache/gravitino/tag/TestTagManager.java | 9 +- .../utils/TestMetadataObjectUtil.java | 124 ++++++++++++++++++ 7 files changed, 195 insertions(+), 12 deletions(-) create mode 100644 core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 06029db45e7..b635c3887c5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -332,7 +332,7 @@ public void close() throws IOException { @Override public List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) throws IOException { - return TagMetaService.getInstance().listAssociatedMetadataObjectIdentsForTag(tagIdent); + return TagMetaService.getInstance().listAssociatedMetadataObjectsForTag(tagIdent); } @Override diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java index 8895fab6da0..71b8275275f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java @@ -217,7 +217,7 @@ public TagEntity getTagForMetadataObject( return POConverters.fromTagPO(tagPO, TagManager.ofTagNamespace(metalake)); } - public List listAssociatedMetadataObjectIdentsForTag(NameIdentifier tagIdent) + public List listAssociatedMetadataObjectsForTag(NameIdentifier tagIdent) throws IOException { String metalakeName = tagIdent.namespace().level(0); String tagName = tagIdent.name(); diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index 061472da79e..1dc65e2d82e 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -31,6 +31,7 @@ import org.apache.gravitino.Entity; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; +import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -210,6 +211,8 @@ public MetadataObject[] listMetadataObjectsForTag(String metalake, String name) tagId, LockType.READ, () -> { + checkMetalakeExists(metalake, entityStore); + try { if (!entityStore.exists(tagId, Entity.EntityType.TAG)) { throw new NoSuchTagException( @@ -238,6 +241,11 @@ public Tag[] listTagsInfoForMetadataObject(String metalake, MetadataObject metad NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + if (!checkAndImportEntity(metalake, metadataObject, GravitinoEnv.getInstance())) { + throw new NotFoundException( + "Failed to list tags for metadata object %s due to not found", metadataObject); + } + return TreeLockUtils.doWithTreeLock( entityIdent, LockType.READ, @@ -262,6 +270,11 @@ public Tag getTagForMetadataObject(String metalake, MetadataObject metadataObjec Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); NameIdentifier tagIdent = ofTagIdent(metalake, name); + if (!checkAndImportEntity(metalake, metadataObject, GravitinoEnv.getInstance())) { + throw new NotFoundException( + "Failed to get tag for metadata object %s due to not found", metadataObject); + } + return TreeLockUtils.doWithTreeLock( entityIdent, LockType.READ, @@ -295,6 +308,11 @@ public String[] associateTagsForMetadataObject( NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + if (!checkAndImportEntity(metalake, metadataObject, GravitinoEnv.getInstance())) { + throw new NotFoundException( + "Failed to associate tags for metadata object %s due to not found", metadataObject); + } + // Remove all the tags that are both set to add and remove Set tagsToAddSet = tagsToAdd == null ? Sets.newHashSet() : Sets.newHashSet(tagsToAdd); Set tagsToRemoveSet = @@ -310,8 +328,6 @@ public String[] associateTagsForMetadataObject( .map(tag -> ofTagIdent(metalake, tag)) .toArray(NameIdentifier[]::new); - // TODO. We need to add a write lock to Tag's namespace to avoid tag alteration and deletion - // during the association operation. return TreeLockUtils.doWithTreeLock( entityIdent, LockType.READ, @@ -359,7 +375,6 @@ private static void checkMetalakeExists(String metalake, EntityStore entityStore } } - @VisibleForTesting public static Namespace ofTagNamespace(String metalake) { return Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.TAG_SCHEMA_NAME); } @@ -407,4 +422,33 @@ private TagEntity updateTagEntity(TagEntity tagEntity, TagChange... changes) { .build()) .build(); } + + // This method will check if the entity is existed explicitly, internally this check will load + // the entity from underlying sources to entity store if not stored, and will allocate an uid + // for this entity, with this uid tags can be associated with this entity. + // This method should be called out of the tree lock, otherwise it will cause deadlock. + @VisibleForTesting + boolean checkAndImportEntity(String metalake, MetadataObject metadataObject, GravitinoEnv env) { + NameIdentifier entityIdent = MetadataObjectUtil.toEntityIdent(metalake, metadataObject); + Entity.EntityType entityType = MetadataObjectUtil.toEntityType(metadataObject); + + switch (entityType) { + case METALAKE: + return env.metalakeDispatcher().metalakeExists(entityIdent); + case CATALOG: + return env.catalogDispatcher().catalogExists(entityIdent); + case SCHEMA: + return env.schemaDispatcher().schemaExists(entityIdent); + case TABLE: + return env.tableDispatcher().tableExists(entityIdent); + case TOPIC: + return env.topicDispatcher().topicExists(entityIdent); + case FILESET: + return env.filesetDispatcher().filesetExists(entityIdent); + case COLUMN: + default: + throw new IllegalArgumentException( + "Unsupported metadata object type: " + metadataObject.type()); + } + } } diff --git a/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java b/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java index 587e1eaa4f7..00545685805 100644 --- a/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java +++ b/core/src/main/java/org/apache/gravitino/utils/MetadataObjectUtil.java @@ -19,9 +19,11 @@ package org.apache.gravitino.utils; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import java.util.Optional; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Entity; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -51,6 +53,8 @@ private MetadataObjectUtil() {} * @throws IllegalArgumentException if the metadata object type is unknown */ public static Entity.EntityType toEntityType(MetadataObject metadataObject) { + Preconditions.checkArgument(metadataObject != null, "metadataObject cannot be null"); + return Optional.ofNullable(TYPE_TO_TYPE_MAP.get(metadataObject.type())) .orElseThrow( () -> @@ -67,6 +71,10 @@ public static Entity.EntityType toEntityType(MetadataObject metadataObject) { * @throws IllegalArgumentException if the metadata object type is unsupported or unknown. */ public static NameIdentifier toEntityIdent(String metalakeName, MetadataObject metadataObject) { + Preconditions.checkArgument( + StringUtils.isNotBlank(metalakeName), "metalakeName cannot be blank"); + Preconditions.checkArgument(metadataObject != null, "metadataObject cannot be null"); + switch (metadataObject.type()) { case METALAKE: return NameIdentifierUtil.ofMetalake(metalakeName); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java index 185b6a2c199..5bb76ad9cc5 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTagMetaService.java @@ -588,7 +588,7 @@ public void testListAssociatedMetadataObjectsForTag() throws IOException { // Test list associated metadata objects for tag2 List metadataObjects = - tagMetaService.listAssociatedMetadataObjectIdentsForTag( + tagMetaService.listAssociatedMetadataObjectsForTag( TagManager.ofTagIdent(metalakeName, "tag2")); Assertions.assertEquals(3, metadataObjects.size()); @@ -603,7 +603,7 @@ public void testListAssociatedMetadataObjectsForTag() throws IOException { // Test list associated metadata objects for tag3 List metadataObjects1 = - tagMetaService.listAssociatedMetadataObjectIdentsForTag( + tagMetaService.listAssociatedMetadataObjectsForTag( TagManager.ofTagIdent(metalakeName, "tag3")); Assertions.assertEquals(3, metadataObjects1.size()); @@ -618,7 +618,7 @@ public void testListAssociatedMetadataObjectsForTag() throws IOException { // Test list associated metadata objects for non-existent tag List metadataObjects2 = - tagMetaService.listAssociatedMetadataObjectIdentsForTag( + tagMetaService.listAssociatedMetadataObjectsForTag( TagManager.ofTagIdent(metalakeName, "tag4")); Assertions.assertEquals(0, metadataObjects2.size()); @@ -629,7 +629,7 @@ public void testListAssociatedMetadataObjectsForTag() throws IOException { false); List metadataObjects3 = - tagMetaService.listAssociatedMetadataObjectIdentsForTag( + tagMetaService.listAssociatedMetadataObjectsForTag( TagManager.ofTagIdent(metalakeName, "tag2")); Assertions.assertEquals(2, metadataObjects3.size()); @@ -643,7 +643,7 @@ public void testListAssociatedMetadataObjectsForTag() throws IOException { NameIdentifier.of(metalakeName, "catalog1", "schema1"), Entity.EntityType.SCHEMA, false); List metadataObjects4 = - tagMetaService.listAssociatedMetadataObjectIdentsForTag( + tagMetaService.listAssociatedMetadataObjectsForTag( TagManager.ofTagIdent(metalakeName, "tag2")); Assertions.assertEquals(1, metadataObjects4.size()); @@ -653,7 +653,7 @@ public void testListAssociatedMetadataObjectsForTag() throws IOException { backend.delete(NameIdentifier.of(metalakeName, "catalog1"), Entity.EntityType.CATALOG, false); List metadataObjects5 = - tagMetaService.listAssociatedMetadataObjectIdentsForTag( + tagMetaService.listAssociatedMetadataObjectsForTag( TagManager.ofTagIdent(metalakeName, "tag2")); Assertions.assertEquals(0, metadataObjects5.size()); diff --git a/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java b/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java index 5154ef7bb94..3dc298cdbf5 100644 --- a/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java +++ b/core/src/test/java/org/apache/gravitino/tag/TestTagManager.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.tag; +import static org.apache.gravitino.Configs.CATALOG_CACHE_EVICTION_INTERVAL_MS; import static org.apache.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; import static org.apache.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; @@ -30,6 +31,8 @@ import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.VERSION_RETENTION_COUNT; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -110,6 +113,7 @@ public static void setUp() throws IOException, IllegalAccessException { Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); Mockito.when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L); + Mockito.when(config.get(CATALOG_CACHE_EVICTION_INTERVAL_MS)).thenReturn(1000L); Mockito.doReturn(100000L).when(config).get(TREE_LOCK_MAX_NODE_IN_MEMORY); Mockito.doReturn(1000L).when(config).get(TREE_LOCK_MIN_NODE_IN_MEMORY); @@ -162,7 +166,10 @@ public static void setUp() throws IOException, IllegalAccessException { .build(); entityStore.put(table, false /* overwritten */); - tagManager = new TagManager(idGenerator, entityStore); + tagManager = spy(new TagManager(idGenerator, entityStore)); + doReturn(true) + .when(tagManager) + .checkAndImportEntity(Mockito.any(), Mockito.any(), Mockito.any()); } @AfterAll diff --git a/core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java b/core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java new file mode 100644 index 00000000000..1de30d16fda --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/utils/TestMetadataObjectUtil.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.utils; + +import org.apache.gravitino.Entity; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.NameIdentifier; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestMetadataObjectUtil { + + @Test + public void testToEntityType() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> MetadataObjectUtil.toEntityType(null), + "metadataObject cannot be null"); + + Assertions.assertEquals( + Entity.EntityType.METALAKE, + MetadataObjectUtil.toEntityType( + MetadataObjects.of(null, "metalake", MetadataObject.Type.METALAKE))); + + Assertions.assertEquals( + Entity.EntityType.CATALOG, + MetadataObjectUtil.toEntityType( + MetadataObjects.of(null, "catalog", MetadataObject.Type.CATALOG))); + + Assertions.assertEquals( + Entity.EntityType.SCHEMA, + MetadataObjectUtil.toEntityType( + MetadataObjects.of("catalog", "schema", MetadataObject.Type.SCHEMA))); + + Assertions.assertEquals( + Entity.EntityType.TABLE, + MetadataObjectUtil.toEntityType( + MetadataObjects.of("catalog.schema", "table", MetadataObject.Type.TABLE))); + + Assertions.assertEquals( + Entity.EntityType.TOPIC, + MetadataObjectUtil.toEntityType( + MetadataObjects.of("catalog.schema", "topic", MetadataObject.Type.TOPIC))); + + Assertions.assertEquals( + Entity.EntityType.FILESET, + MetadataObjectUtil.toEntityType( + MetadataObjects.of("catalog.schema", "fileset", MetadataObject.Type.FILESET))); + + Assertions.assertEquals( + Entity.EntityType.COLUMN, + MetadataObjectUtil.toEntityType( + MetadataObjects.of("catalog.schema.table", "column", MetadataObject.Type.COLUMN))); + } + + @Test + public void testToEntityIdent() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> MetadataObjectUtil.toEntityIdent(null, null), + "metadataName cannot be blank"); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> MetadataObjectUtil.toEntityIdent("metalake", null), + "metadataObject cannot be null"); + + Assertions.assertEquals( + NameIdentifier.of("metalake"), + MetadataObjectUtil.toEntityIdent( + "metalake", MetadataObjects.of(null, "metalake", MetadataObject.Type.METALAKE))); + + Assertions.assertEquals( + NameIdentifier.of("metalake", "catalog"), + MetadataObjectUtil.toEntityIdent( + "metalake", MetadataObjects.of(null, "catalog", MetadataObject.Type.CATALOG))); + + Assertions.assertEquals( + NameIdentifier.of("metalake", "catalog", "schema"), + MetadataObjectUtil.toEntityIdent( + "metalake", MetadataObjects.of("catalog", "schema", MetadataObject.Type.SCHEMA))); + + Assertions.assertEquals( + NameIdentifier.of("metalake", "catalog", "schema", "table"), + MetadataObjectUtil.toEntityIdent( + "metalake", MetadataObjects.of("catalog.schema", "table", MetadataObject.Type.TABLE))); + + Assertions.assertEquals( + NameIdentifier.of("metalake", "catalog", "schema", "topic"), + MetadataObjectUtil.toEntityIdent( + "metalake", MetadataObjects.of("catalog.schema", "topic", MetadataObject.Type.TOPIC))); + + Assertions.assertEquals( + NameIdentifier.of("metalake", "catalog", "schema", "fileset"), + MetadataObjectUtil.toEntityIdent( + "metalake", + MetadataObjects.of("catalog.schema", "fileset", MetadataObject.Type.FILESET))); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + MetadataObjectUtil.toEntityIdent( + "metalake", + MetadataObjects.of("catalog.schema.table", "column", MetadataObject.Type.COLUMN)), + "Cannot convert column metadata object to entity identifier: catalog.schema.table.column"); + } +} From 3efd7f440d1d8731570335d4beeb456960f7c30f Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Thu, 18 Jul 2024 14:42:59 +0800 Subject: [PATCH 26/27] Address the comments --- .../org/apache/gravitino/MetadataObject.java | 17 +- .../responses/MetadataObjectListResponse.java | 6 +- .../dto/responses/NameListResponse.java | 7 + .../server/web/rest/TagOperations.java | 84 +++---- .../server/web/rest/TestTagOperations.java | 230 ++++++++---------- 5 files changed, 156 insertions(+), 188 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/MetadataObject.java b/api/src/main/java/org/apache/gravitino/MetadataObject.java index a505b065aaa..4636491c8f4 100644 --- a/api/src/main/java/org/apache/gravitino/MetadataObject.java +++ b/api/src/main/java/org/apache/gravitino/MetadataObject.java @@ -19,6 +19,7 @@ package org.apache.gravitino; +import java.util.Locale; import javax.annotation.Nullable; import org.apache.gravitino.annotation.Unstable; @@ -57,7 +58,21 @@ enum Type { */ TOPIC, /** A column is a sub-collection of the table that represents a group of same type data. */ - COLUMN + COLUMN; + + /** + * Convert the string type to the Type enum. + * + * @param type The string type. + * @return The Type enum. + */ + public static Type toType(String type) { + try { + return Type.valueOf(type.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown type: " + type); + } + } } /** diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java index 91a1f61eff2..664f1eb5bec 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java @@ -65,7 +65,9 @@ public void validate() throws IllegalArgumentException { .forEach( object -> Preconditions.checkArgument( - object != null && StringUtils.isNoneBlank(object.name()), - "metadataObject must not be null and empty")); + object != null + && StringUtils.isNoneBlank(object.name()) + && object.type() != null, + "metadataObject must not be null and it's field cannot null or empty")); } } diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java index e37316af708..0a4c0682de3 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java @@ -20,9 +20,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import java.util.Arrays; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.apache.commons.lang3.StringUtils; /** Represents a response for a list of entity names. */ @Getter @@ -55,5 +57,10 @@ public void validate() throws IllegalArgumentException { super.validate(); Preconditions.checkArgument(names != null, "\"names\" must not be null"); + Arrays.stream(names) + .forEach( + name -> + Preconditions.checkArgument( + StringUtils.isNoneBlank(name), "name must not be null")); } } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java index b27ee7dcf46..abdceb8ab58 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java @@ -24,7 +24,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; @@ -32,6 +31,7 @@ import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -164,7 +164,7 @@ public Response getTag(@PathParam("metalake") String metalake, @PathParam("tag") } } - @POST + @PUT @Path("{tag}") @Produces("application/vnd.gravitino.v1+json") @Timed(name = "alter-tag." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @@ -226,7 +226,7 @@ public Response deleteTag(@PathParam("metalake") String metalake, @PathParam("ta @Produces("application/vnd.gravitino.v1+json") @Timed(name = "list-object-tags." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "list-object-tags", absolute = true) - public Response listTagsForMetdataObject( + public Response listTagsForMetadataObject( @PathParam("metalake") String metalake, @PathParam("type") String type, @PathParam("fullName") String fullName, @@ -242,33 +242,34 @@ public Response listTagsForMetdataObject( return Utils.doAs( httpRequest, () -> { - MetadataObject object = MetadataObjects.parse(fullName, toType(type)); + MetadataObject object = + MetadataObjects.parse(fullName, MetadataObject.Type.toType(type)); + + List tags = Lists.newArrayList(); + Tag[] nonInheritedTags = tagManager.listTagsInfoForMetadataObject(metalake, object); + if (ArrayUtils.isNotEmpty(nonInheritedTags)) { + Collections.addAll( + tags, + Arrays.stream(nonInheritedTags) + .map(t -> DTOConverters.toDTO(t, Optional.of(false))) + .toArray(TagDTO[]::new)); + } - if (verbose) { - List tags = Lists.newArrayList(); - Tag[] nonInheritedTags = tagManager.listTagsInfoForMetadataObject(metalake, object); - if (ArrayUtils.isNotEmpty(nonInheritedTags)) { + MetadataObject parentObject = MetadataObjects.parent(object); + while (parentObject != null) { + Tag[] inheritedTags = + tagManager.listTagsInfoForMetadataObject(metalake, parentObject); + if (ArrayUtils.isNotEmpty(inheritedTags)) { Collections.addAll( tags, - Arrays.stream(nonInheritedTags) - .map(t -> DTOConverters.toDTO(t, Optional.of(false))) + Arrays.stream(inheritedTags) + .map(t -> DTOConverters.toDTO(t, Optional.of(true))) .toArray(TagDTO[]::new)); } + parentObject = MetadataObjects.parent(parentObject); + } - MetadataObject parentObject = MetadataObjects.parent(object); - while (parentObject != null) { - Tag[] inheritedTags = - tagManager.listTagsInfoForMetadataObject(metalake, parentObject); - if (ArrayUtils.isNotEmpty(inheritedTags)) { - Collections.addAll( - tags, - Arrays.stream(inheritedTags) - .map(t -> DTOConverters.toDTO(t, Optional.of(true))) - .toArray(TagDTO[]::new)); - } - parentObject = MetadataObjects.parent(parentObject); - } - + if (verbose) { LOG.info( "List {} tags info for object type: {}, full name: {} under metalake: {}", tags.size(), @@ -278,30 +279,15 @@ public Response listTagsForMetdataObject( return Utils.ok(new TagListResponse(tags.toArray(new TagDTO[0]))); } else { - List tagNames = Lists.newArrayList(); - String[] nonInheritedTagNames = - tagManager.listTagsForMetadataObject(metalake, object); - if (ArrayUtils.isNotEmpty(nonInheritedTagNames)) { - Collections.addAll(tagNames, nonInheritedTagNames); - } - - MetadataObject parentObject = MetadataObjects.parent(object); - while (parentObject != null) { - String[] inheritedTagNames = - tagManager.listTagsForMetadataObject(metalake, parentObject); - if (ArrayUtils.isNotEmpty(inheritedTagNames)) { - Collections.addAll(tagNames, inheritedTagNames); - } - parentObject = MetadataObjects.parent(parentObject); - } + String[] tagNames = tags.stream().map(TagDTO::name).toArray(String[]::new); LOG.info( "List {} tags for object type: {}, full name: {} under metalake: {}", - tagNames.stream(), + tagNames.length, type, fullName, metalake); - return Utils.ok(new NameListResponse(tagNames.toArray(new String[0]))); + return Utils.ok(new NameListResponse(tagNames)); } }); @@ -331,7 +317,8 @@ public Response getTagForObject( return Utils.doAs( httpRequest, () -> { - MetadataObject object = MetadataObjects.parse(fullName, toType(type)); + MetadataObject object = + MetadataObjects.parse(fullName, MetadataObject.Type.toType(type)); Optional tag = getTagForObject(metalake, object, tagName); Optional tagDTO = tag.map(t -> DTOConverters.toDTO(t, Optional.of(false))); @@ -428,7 +415,8 @@ public Response associateTagsForObject( httpRequest, () -> { request.validate(); - MetadataObject object = MetadataObjects.parse(fullName, toType(type)); + MetadataObject object = + MetadataObjects.parse(fullName, MetadataObject.Type.toType(type)); String[] tagNames = tagManager.associateTagsForMetadataObject( metalake, object, request.getTagsToAdd(), request.getTagsToRemove()); @@ -448,14 +436,6 @@ public Response associateTagsForObject( } } - private MetadataObject.Type toType(String type) { - try { - return MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid type: " + type); - } - } - private Optional getTagForObject(String metalake, MetadataObject object, String tagName) { try { return Optional.ofNullable(tagManager.getTagForMetadataObject(metalake, object, tagName)); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java index 8dc2eb22d91..23b87b60d28 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestTagOperations.java @@ -409,7 +409,7 @@ public void testAlterTag() { .path("tag1") .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") - .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); @@ -430,7 +430,7 @@ public void testAlterTag() { .path("tag1") .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") - .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); @@ -446,7 +446,7 @@ public void testAlterTag() { .path("tag1") .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") - .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); Assertions.assertEquals( Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); @@ -508,131 +508,13 @@ public void testDeleteTag() { @Test public void testListTagsForObject() { - String[] catalogTags = new String[] {"tag1", "tag2"}; MetadataObject catalog = MetadataObjects.parse("object1", MetadataObject.Type.CATALOG); - when(tagManager.listTagsForMetadataObject(metalake, catalog)).thenReturn(catalogTags); - - String[] schemaTags = new String[] {"tag3", "tag4"}; MetadataObject schema = MetadataObjects.parse("object1.object2", MetadataObject.Type.SCHEMA); - when(tagManager.listTagsForMetadataObject(metalake, schema)).thenReturn(schemaTags); - - String[] tableTags = new String[] {"tag5", "tag6"}; MetadataObject table = MetadataObjects.parse("object1.object2.object3", MetadataObject.Type.TABLE); - when(tagManager.listTagsForMetadataObject(metalake, table)).thenReturn(tableTags); - - String[] columnTags = new String[] {"tag7", "tag8"}; MetadataObject column = MetadataObjects.parse("object1.object2.object3.object4", MetadataObject.Type.COLUMN); - when(tagManager.listTagsForMetadataObject(metalake, column)).thenReturn(columnTags); - - // Test catalog tags - Response response = - target(tagPath(metalake)) - .path(catalog.type().toString()) - .path(catalog.fullName()) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); - - NameListResponse nameListResponse = response.readEntity(NameListResponse.class); - Assertions.assertEquals(0, nameListResponse.getCode()); - Assertions.assertArrayEquals(catalogTags, nameListResponse.getNames()); - - // Test schema tags - Response response1 = - target(tagPath(metalake)) - .path(schema.type().toString()) - .path(schema.fullName()) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response1.getStatus()); - - NameListResponse nameListResponse1 = response1.readEntity(NameListResponse.class); - Assertions.assertEquals(0, nameListResponse1.getCode()); - - Set expected = Sets.newHashSet(catalogTags); - expected.addAll(Sets.newHashSet(schemaTags)); - Assertions.assertEquals(expected, Sets.newHashSet(nameListResponse1.getNames())); - - // Test table tags - Response response2 = - target(tagPath(metalake)) - .path(table.type().toString()) - .path(table.fullName()) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response2.getStatus()); - - NameListResponse nameListResponse2 = response2.readEntity(NameListResponse.class); - Assertions.assertEquals(0, nameListResponse2.getCode()); - - Set expected1 = Sets.newHashSet(catalogTags); - expected1.addAll(Sets.newHashSet(schemaTags)); - expected1.addAll(Sets.newHashSet(tableTags)); - Assertions.assertEquals(expected1, Sets.newHashSet(nameListResponse2.getNames())); - - // Test column tags - Response response3 = - target(tagPath(metalake)) - .path(column.type().toString()) - .path(column.fullName()) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response3.getStatus()); - - NameListResponse nameListResponse3 = response3.readEntity(NameListResponse.class); - Assertions.assertEquals(0, nameListResponse3.getCode()); - - Set expected2 = Sets.newHashSet(catalogTags); - expected2.addAll(Sets.newHashSet(schemaTags)); - expected2.addAll(Sets.newHashSet(tableTags)); - expected2.addAll(Sets.newHashSet(columnTags)); - Assertions.assertEquals(expected2, Sets.newHashSet(nameListResponse3.getNames())); - - // Test with null tags - when(tagManager.listTagsForMetadataObject(metalake, catalog)).thenReturn(null); - Response response4 = - target(tagPath(metalake)) - .path(catalog.type().toString()) - .path(catalog.fullName()) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response4.getStatus()); - - NameListResponse nameListResponse4 = response4.readEntity(NameListResponse.class); - Assertions.assertEquals(0, nameListResponse4.getCode()); - - Assertions.assertEquals(0, nameListResponse4.getNames().length); - - Response response5 = - target(tagPath(metalake)) - .path(schema.type().toString()) - .path(schema.fullName()) - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .get(); - - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response5.getStatus()); - - NameListResponse nameListResponse5 = response5.readEntity(NameListResponse.class); - Assertions.assertEquals(0, nameListResponse5.getCode()); - - Set expected3 = Sets.newHashSet(schemaTags); - Assertions.assertEquals(expected3, Sets.newHashSet(nameListResponse5.getNames())); - // Test with "details" = true Tag[] catalogTagInfos = new Tag[] { TagEntity.builder().withName("tag1").withId(1L).withAuditInfo(testAuditInfo1).build() @@ -658,7 +540,7 @@ public void testListTagsForObject() { when(tagManager.listTagsInfoForMetadataObject(metalake, column)).thenReturn(columnTagInfos); // Test catalog tags - Response response6 = + Response response = target(tagPath(metalake)) .path(catalog.type().toString()) .path(catalog.fullName()) @@ -667,9 +549,9 @@ public void testListTagsForObject() { .accept("application/vnd.gravitino.v1+json") .get(); - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response6.getStatus()); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - TagListResponse tagListResponse = response6.readEntity(TagListResponse.class); + TagListResponse tagListResponse = response.readEntity(TagListResponse.class); Assertions.assertEquals(0, tagListResponse.getCode()); Assertions.assertEquals(catalogTagInfos.length, tagListResponse.getTags().length); @@ -680,8 +562,25 @@ public void testListTagsForObject() { Assertions.assertTrue(resultTags.containsKey("tag1")); Assertions.assertFalse(resultTags.get("tag1").inherited().get()); + Response response1 = + target(tagPath(metalake)) + .path(catalog.type().toString()) + .path(catalog.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response1.getStatus()); + + NameListResponse nameListResponse = response1.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse.getCode()); + Assertions.assertEquals(catalogTagInfos.length, nameListResponse.getNames().length); + Assertions.assertArrayEquals( + Arrays.stream(catalogTagInfos).map(Tag::name).toArray(String[]::new), + nameListResponse.getNames()); + // Test schema tags - Response response7 = + Response response2 = target(tagPath(metalake)) .path(schema.type().toString()) .path(schema.fullName()) @@ -690,9 +589,9 @@ public void testListTagsForObject() { .accept("application/vnd.gravitino.v1+json") .get(); - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response7.getStatus()); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response2.getStatus()); - TagListResponse tagListResponse1 = response7.readEntity(TagListResponse.class); + TagListResponse tagListResponse1 = response2.readEntity(TagListResponse.class); Assertions.assertEquals(0, tagListResponse1.getCode()); Assertions.assertEquals( schemaTagInfos.length + catalogTagInfos.length, tagListResponse1.getTags().length); @@ -707,8 +606,26 @@ public void testListTagsForObject() { Assertions.assertTrue(resultTags1.get("tag1").inherited().get()); Assertions.assertFalse(resultTags1.get("tag3").inherited().get()); + Response response3 = + target(tagPath(metalake)) + .path(schema.type().toString()) + .path(schema.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response3.getStatus()); + + NameListResponse nameListResponse1 = response3.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse1.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length, nameListResponse1.getNames().length); + Set resultNames = Sets.newHashSet(nameListResponse1.getNames()); + Assertions.assertTrue(resultNames.contains("tag1")); + Assertions.assertTrue(resultNames.contains("tag3")); + // Test table tags - Response response8 = + Response response4 = target(tagPath(metalake)) .path(table.type().toString()) .path(table.fullName()) @@ -717,9 +634,9 @@ public void testListTagsForObject() { .accept("application/vnd.gravitino.v1+json") .get(); - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response8.getStatus()); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response4.getStatus()); - TagListResponse tagListResponse2 = response8.readEntity(TagListResponse.class); + TagListResponse tagListResponse2 = response4.readEntity(TagListResponse.class); Assertions.assertEquals(0, tagListResponse2.getCode()); Assertions.assertEquals( schemaTagInfos.length + catalogTagInfos.length + tableTagInfos.length, @@ -737,8 +654,29 @@ public void testListTagsForObject() { Assertions.assertTrue(resultTags2.get("tag3").inherited().get()); Assertions.assertFalse(resultTags2.get("tag5").inherited().get()); + Response response5 = + target(tagPath(metalake)) + .path(table.type().toString()) + .path(table.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response5.getStatus()); + + NameListResponse nameListResponse2 = response5.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse2.getCode()); + Assertions.assertEquals( + schemaTagInfos.length + catalogTagInfos.length + tableTagInfos.length, + nameListResponse2.getNames().length); + + Set resultNames1 = Sets.newHashSet(nameListResponse2.getNames()); + Assertions.assertTrue(resultNames1.contains("tag1")); + Assertions.assertTrue(resultNames1.contains("tag3")); + Assertions.assertTrue(resultNames1.contains("tag5")); + // Test column tags - Response response9 = + Response response6 = target(tagPath(metalake)) .path(column.type().toString()) .path(column.fullName()) @@ -747,9 +685,9 @@ public void testListTagsForObject() { .accept("application/vnd.gravitino.v1+json") .get(); - Assertions.assertEquals(Response.Status.OK.getStatusCode(), response9.getStatus()); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response6.getStatus()); - TagListResponse tagListResponse3 = response9.readEntity(TagListResponse.class); + TagListResponse tagListResponse3 = response6.readEntity(TagListResponse.class); Assertions.assertEquals(0, tagListResponse3.getCode()); Assertions.assertEquals( schemaTagInfos.length @@ -771,6 +709,32 @@ public void testListTagsForObject() { Assertions.assertTrue(resultTags3.get("tag3").inherited().get()); Assertions.assertTrue(resultTags3.get("tag5").inherited().get()); Assertions.assertFalse(resultTags3.get("tag7").inherited().get()); + + Response response7 = + target(tagPath(metalake)) + .path(column.type().toString()) + .path(column.fullName()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response7.getStatus()); + + NameListResponse nameListResponse3 = response7.readEntity(NameListResponse.class); + Assertions.assertEquals(0, nameListResponse3.getCode()); + + Assertions.assertEquals( + schemaTagInfos.length + + catalogTagInfos.length + + tableTagInfos.length + + columnTagInfos.length, + nameListResponse3.getNames().length); + + Set resultNames2 = Sets.newHashSet(nameListResponse3.getNames()); + Assertions.assertTrue(resultNames2.contains("tag1")); + Assertions.assertTrue(resultNames2.contains("tag3")); + Assertions.assertTrue(resultNames2.contains("tag5")); + Assertions.assertTrue(resultNames2.contains("tag7")); } @Test From b31490af94f66a07a4b8fa78be7b8a20e5da67b7 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Mon, 22 Jul 2024 18:19:01 +0800 Subject: [PATCH 27/27] Address the comments --- .../org/apache/gravitino/MetadataObject.java | 17 +---------------- .../dto/requests/TagsAssociateRequest.java | 8 ++++++-- .../responses/MetadataObjectListResponse.java | 2 +- .../dto/responses/NameListResponse.java | 3 +-- .../dto/responses/TagListResponse.java | 3 +++ .../gravitino/dto/tag/MetadataObjectDTO.java | 2 +- .../server/web/rest/TagOperations.java | 10 +++++++--- 7 files changed, 20 insertions(+), 25 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/MetadataObject.java b/api/src/main/java/org/apache/gravitino/MetadataObject.java index 4636491c8f4..a505b065aaa 100644 --- a/api/src/main/java/org/apache/gravitino/MetadataObject.java +++ b/api/src/main/java/org/apache/gravitino/MetadataObject.java @@ -19,7 +19,6 @@ package org.apache.gravitino; -import java.util.Locale; import javax.annotation.Nullable; import org.apache.gravitino.annotation.Unstable; @@ -58,21 +57,7 @@ enum Type { */ TOPIC, /** A column is a sub-collection of the table that represents a group of same type data. */ - COLUMN; - - /** - * Convert the string type to the Type enum. - * - * @param type The string type. - * @return The Type enum. - */ - public static Type toType(String type) { - try { - return Type.valueOf(type.toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Unknown type: " + type); - } - } + COLUMN } /** diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/TagsAssociateRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/TagsAssociateRequest.java index 6a3df8bb352..fe202b1d608 100644 --- a/common/src/main/java/org/apache/gravitino/dto/requests/TagsAssociateRequest.java +++ b/common/src/main/java/org/apache/gravitino/dto/requests/TagsAssociateRequest.java @@ -61,17 +61,21 @@ public TagsAssociateRequest() { */ @Override public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + tagsToAdd != null || tagsToRemove != null, + "tagsToAdd and tagsToRemove cannot both be null"); + if (tagsToAdd != null) { for (String tag : tagsToAdd) { Preconditions.checkArgument( - StringUtils.isNotBlank(tag), "tagsToAdd must not contain null or empty strings"); + StringUtils.isNotBlank(tag), "tagsToAdd must not contain null or empty tag names"); } } if (tagsToRemove != null) { for (String tag : tagsToRemove) { Preconditions.checkArgument( - StringUtils.isNotBlank(tag), "tagsToRemove must not contain null or empty strings"); + StringUtils.isNotBlank(tag), "tagsToRemove must not contain null or empty tag names"); } } } diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java index 664f1eb5bec..67b26a5f531 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/MetadataObjectListResponse.java @@ -66,7 +66,7 @@ public void validate() throws IllegalArgumentException { object -> Preconditions.checkArgument( object != null - && StringUtils.isNoneBlank(object.name()) + && StringUtils.isNotBlank(object.name()) && object.type() != null, "metadataObject must not be null and it's field cannot null or empty")); } diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java index 0a4c0682de3..0728165c5e8 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/NameListResponse.java @@ -60,7 +60,6 @@ public void validate() throws IllegalArgumentException { Arrays.stream(names) .forEach( name -> - Preconditions.checkArgument( - StringUtils.isNoneBlank(name), "name must not be null")); + Preconditions.checkArgument(StringUtils.isNotBlank(name), "name must not be null")); } } diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java index 428f3998680..93c324dfe45 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/TagListResponse.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import java.util.Arrays; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; @@ -58,5 +59,7 @@ public void validate() throws IllegalArgumentException { super.validate(); Preconditions.checkArgument(tags != null, "\"tags\" must not be null"); + Arrays.stream(tags) + .forEach(t -> Preconditions.checkArgument(t != null, "tag must not be null")); } } diff --git a/common/src/main/java/org/apache/gravitino/dto/tag/MetadataObjectDTO.java b/common/src/main/java/org/apache/gravitino/dto/tag/MetadataObjectDTO.java index bd36fc294ec..d09815b9e2c 100644 --- a/common/src/main/java/org/apache/gravitino/dto/tag/MetadataObjectDTO.java +++ b/common/src/main/java/org/apache/gravitino/dto/tag/MetadataObjectDTO.java @@ -57,7 +57,7 @@ public String getFullName() { } /** - * Sets the full name of the metadata object. + * Sets the full name of the metadata object. Only used by Jackson deserializer. * * @param fullName The full name of the metadata object. */ diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java index abdceb8ab58..4a3abe7f96c 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/TagOperations.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; @@ -243,7 +244,8 @@ public Response listTagsForMetadataObject( httpRequest, () -> { MetadataObject object = - MetadataObjects.parse(fullName, MetadataObject.Type.toType(type)); + MetadataObjects.parse( + fullName, MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT))); List tags = Lists.newArrayList(); Tag[] nonInheritedTags = tagManager.listTagsInfoForMetadataObject(metalake, object); @@ -318,7 +320,8 @@ public Response getTagForObject( httpRequest, () -> { MetadataObject object = - MetadataObjects.parse(fullName, MetadataObject.Type.toType(type)); + MetadataObjects.parse( + fullName, MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT))); Optional tag = getTagForObject(metalake, object, tagName); Optional tagDTO = tag.map(t -> DTOConverters.toDTO(t, Optional.of(false))); @@ -416,7 +419,8 @@ public Response associateTagsForObject( () -> { request.validate(); MetadataObject object = - MetadataObjects.parse(fullName, MetadataObject.Type.toType(type)); + MetadataObjects.parse( + fullName, MetadataObject.Type.valueOf(type.toUpperCase(Locale.ROOT))); String[] tagNames = tagManager.associateTagsForMetadataObject( metalake, object, request.getTagsToAdd(), request.getTagsToRemove());