From e2d1fd0091cf89d6ccbcc386999832f9ad82c780 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 17 Apr 2024 16:52:06 +0800 Subject: [PATCH] [#2856] fix(core): Fix the namespace missing issue when get entities from kv storage (#2971) ### What changes were proposed in this pull request? This PR fixes the namespace missing issue when entities are gotten from kv storage. ### Why are the changes needed? Fix: #2856 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Modified UTs. --- build.gradle.kts | 3 +- .../com/datastrato/gravitino/EntitySerDe.java | 10 +- .../gravitino/catalog/CatalogManager.java | 2 +- .../gravitino/meta/CatalogEntity.java | 14 +-- .../gravitino/meta/FilesetEntity.java | 1 + .../gravitino/meta/GroupEntity.java | 1 + .../datastrato/gravitino/meta/RoleEntity.java | 1 + .../gravitino/meta/SchemaEntity.java | 1 + .../gravitino/meta/TableEntity.java | 1 + .../gravitino/meta/TopicEntity.java | 1 + .../datastrato/gravitino/meta/UserEntity.java | 1 + .../gravitino/proto/AuditInfoSerDe.java | 3 +- .../gravitino/proto/BaseMetalakeSerDe.java | 5 +- .../gravitino/proto/CatalogEntitySerDe.java | 6 +- .../gravitino/proto/FilesetEntitySerDe.java | 6 +- .../gravitino/proto/GroupEntitySerDe.java | 6 +- .../gravitino/proto/ProtoEntitySerDe.java | 10 +- .../gravitino/proto/ProtoSerDe.java | 4 +- .../gravitino/proto/RoleEntitySerDe.java | 6 +- .../gravitino/proto/SchemaEntitySerDe.java | 6 +- .../gravitino/proto/TableEntitySerDe.java | 6 +- .../gravitino/proto/TopicEntitySerDe.java | 6 +- .../gravitino/proto/UserEntitySerDe.java | 6 +- .../gravitino/storage/kv/KvEntityStore.java | 6 +- .../gravitino/proto/TestEntityProtoSerDe.java | 98 ++++++++++++++----- 25 files changed, 142 insertions(+), 68 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 7d383ced5ff..c70b4fc7cf3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -539,7 +539,8 @@ tasks.rat { "**/LICENSE.*", "**/NOTICE.*", "ROADMAP.md", - "clients/client-python/.pytest_cache/*" + "clients/client-python/.pytest_cache/*", + "clients/client-python/gravitino.egg-info/*" ) // Add .gitignore excludes to the Apache Rat exclusion list. diff --git a/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java index 1ebcb9c0663..103ed34ed83 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/EntitySerDe.java @@ -24,15 +24,17 @@ public interface EntitySerDe { * * @param bytes the byte array to deserialize * @param clazz the class of the entity + * @param namespace the namespace to use when deserializing the entity * @return the deserialized entity * @param The type of entity * @throws IOException if the deserialization fails */ - default T deserialize(byte[] bytes, Class clazz) throws IOException { + default T deserialize(byte[] bytes, Class clazz, Namespace namespace) + throws IOException { ClassLoader loader = Optional.ofNullable(Thread.currentThread().getContextClassLoader()) .orElse(getClass().getClassLoader()); - return deserialize(bytes, clazz, loader); + return deserialize(bytes, clazz, loader, namespace); } /** @@ -41,10 +43,12 @@ default T deserialize(byte[] bytes, Class clazz) throws IO * @param bytes the byte array to deserialize * @param clazz the class of the entity * @param classLoader the class loader to use + * @param namespace the namespace to use when deserializing the entity * @return the deserialized entity * @param The type of entity * @throws IOException if the deserialization fails */ - T deserialize(byte[] bytes, Class clazz, ClassLoader classLoader) + T deserialize( + byte[] bytes, Class clazz, ClassLoader classLoader, Namespace namespace) throws IOException; } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 884a5adfff0..266786da59d 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -528,7 +528,7 @@ private void checkMetalakeExists(NameIdentifier ident) throws NoSuchMetalakeExce private CatalogWrapper loadCatalogInternal(NameIdentifier ident) throws NoSuchCatalogException { try { CatalogEntity entity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); - return createCatalogWrapper(entity.withNamespace(ident.namespace())); + return createCatalogWrapper(entity); } catch (NoSuchEntityException ne) { LOG.warn("Catalog {} does not exist", ident, ne); diff --git a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java index 2a2d7757d2c..e1ce59ccace 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/CatalogEntity.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.connector.CatalogInfo; -import com.datastrato.gravitino.proto.CatalogEntitySerDe; import com.google.common.base.Objects; import java.util.Collections; import java.util.HashMap; @@ -124,18 +123,6 @@ public CatalogInfo toCatalogInfo() { return new CatalogInfo(id, name, type, provider, comment, properties, auditInfo, namespace); } - /** - * Sets the namespace of the catalog entity. because the {@link CatalogEntitySerDe} does not - * serialize the namespace field - * - * @param namespace the namespace of the catalog entity. - * @return the instance of the source catalog entity. - */ - public CatalogEntity withNamespace(Namespace namespace) { - this.namespace = namespace; - return this; - } - /** Builder class for creating instances of {@link CatalogEntity}. */ public static class Builder { @@ -263,6 +250,7 @@ public boolean equals(Object o) { CatalogEntity that = (CatalogEntity) o; return Objects.equal(id, that.id) && Objects.equal(name, that.name) + && Objects.equal(namespace, that.namespace) && type == that.type && Objects.equal(provider, that.provider) && Objects.equal(comment, that.comment) diff --git a/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java index ae6ccb97f48..0f8726a313b 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/FilesetEntity.java @@ -166,6 +166,7 @@ public boolean equals(Object o) { FilesetEntity that = (FilesetEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(comment, that.comment) && Objects.equals(type, that.type) && Objects.equals(storageLocation, that.storageLocation) diff --git a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java index 9ae0d8cee07..e00b59b908b 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/GroupEntity.java @@ -120,6 +120,7 @@ public boolean equals(Object o) { GroupEntity that = (GroupEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(roles, that.roles); } diff --git a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java index b4642f3e5db..8e9ec407f8d 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java @@ -152,6 +152,7 @@ public boolean equals(Object o) { RoleEntity that = (RoleEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties) && Objects.equals(securableObject, that.securableObject) diff --git a/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java index 9805af07a1c..46e747992ea 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/SchemaEntity.java @@ -145,6 +145,7 @@ public boolean equals(Object o) { SchemaEntity schema = (SchemaEntity) o; return Objects.equal(id, schema.id) && Objects.equal(name, schema.name) + && Objects.equal(namespace, schema.namespace) && Objects.equal(comment, schema.comment) && Objects.equal(properties, schema.properties) && Objects.equal(auditInfo, schema.auditInfo); diff --git a/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java index 5275f9dac81..37d1ee5a984 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/TableEntity.java @@ -109,6 +109,7 @@ public boolean equals(Object o) { TableEntity baseTable = (TableEntity) o; return Objects.equal(id, baseTable.id) && Objects.equal(name, baseTable.name) + && Objects.equal(namespace, baseTable.namespace) && Objects.equal(auditInfo, baseTable.auditInfo); } diff --git a/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java index 8b0a3180314..898e9b35bba 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/TopicEntity.java @@ -135,6 +135,7 @@ public boolean equals(Object o) { TopicEntity that = (TopicEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(comment, that.comment) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties); diff --git a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java index 11a6fe6978a..933ecb1eb5e 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/UserEntity.java @@ -124,6 +124,7 @@ public boolean equals(Object o) { UserEntity that = (UserEntity) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name) + && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(roles, that.roles); } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java index 45058f00979..7891af5ccc5 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/AuditInfoSerDe.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import java.util.Optional; /** A class for serializing and deserializing AuditInfo objects. */ @@ -41,7 +42,7 @@ public AuditInfo serialize(com.datastrato.gravitino.meta.AuditInfo auditInfo) { * @return The deserialized AuditInfo object. */ @Override - public com.datastrato.gravitino.meta.AuditInfo deserialize(AuditInfo p) { + public com.datastrato.gravitino.meta.AuditInfo deserialize(AuditInfo p, Namespace namespace) { com.datastrato.gravitino.meta.AuditInfo.Builder builder = com.datastrato.gravitino.meta.AuditInfo.builder(); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java index ad90e5fc152..0f9f09153dd 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/BaseMetalakeSerDe.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.AuditInfo; /** A class for serializing and deserializing BaseMetalake objects. */ @@ -52,13 +53,13 @@ public Metalake serialize(com.datastrato.gravitino.meta.BaseMetalake baseMetalak * @return The deserialized BaseMetalake object. */ @Override - public com.datastrato.gravitino.meta.BaseMetalake deserialize(Metalake p) { + public com.datastrato.gravitino.meta.BaseMetalake deserialize(Metalake p, Namespace namespace) { com.datastrato.gravitino.meta.BaseMetalake.Builder builder = com.datastrato.gravitino.meta.BaseMetalake.builder(); builder .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java index 2aafa3d3d85..382761da4d9 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/CatalogEntitySerDe.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.CatalogEntity; @@ -49,13 +50,14 @@ public Catalog serialize(CatalogEntity catalogEntity) { * @return The deserialized CatalogEntity object. */ @Override - public CatalogEntity deserialize(Catalog p) { + public CatalogEntity deserialize(Catalog p, Namespace namespace) { CatalogEntity.Builder builder = CatalogEntity.builder(); builder .withId(p.getId()) .withName(p.getName()) + .withNamespace(namespace) .withProvider(p.getProvider()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java index ef724ecb41a..24680452739 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/FilesetEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.FilesetEntity; public class FilesetEntitySerDe implements ProtoSerDe { @@ -31,13 +32,14 @@ public Fileset serialize(FilesetEntity filesetEntity) { } @Override - public FilesetEntity deserialize(Fileset p) { + public FilesetEntity deserialize(Fileset p, Namespace namespace) { FilesetEntity.Builder builder = FilesetEntity.builder() .withId(p.getId()) .withName(p.getName()) + .withNamespace(namespace) .withStorageLocation(p.getStorageLocation()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)) .withFilesetType( com.datastrato.gravitino.file.Fileset.Type.valueOf(p.getType().name())); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java index 083b69029d8..e1fa6ab50c2 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/GroupEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.GroupEntity; import java.util.Collection; @@ -25,12 +26,13 @@ public Group serialize(GroupEntity groupEntity) { } @Override - public GroupEntity deserialize(Group group) { + public GroupEntity deserialize(Group group, Namespace namespace) { GroupEntity.Builder builder = GroupEntity.builder() .withId(group.getId()) .withName(group.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(group.getAuditInfo(), namespace)); if (group.getRolesCount() > 0) { builder.withRoles(group.getRolesList()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java index b826b4cedb3..894607d7dbf 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntitySerDe; +import com.datastrato.gravitino.Namespace; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.protobuf.Any; @@ -91,7 +92,8 @@ public byte[] serialize(T t) throws IOException { } @Override - public T deserialize(byte[] bytes, Class clazz, ClassLoader classLoader) + public T deserialize( + byte[] bytes, Class clazz, ClassLoader classLoader, Namespace namespace) throws IOException { Any any = Any.parseFrom(bytes); Class protoClass = getProtoClass(clazz, classLoader); @@ -101,7 +103,7 @@ public T deserialize(byte[] bytes, Class clazz, ClassLoade } Message anyMessage = any.unpack(protoClass); - return fromProto(anyMessage, clazz, classLoader); + return fromProto(anyMessage, clazz, classLoader, namespace); } private ProtoSerDe getProtoSerde( @@ -151,9 +153,9 @@ private M toProto(T t, ClassLoader classLo } private T fromProto( - M m, Class entityClass, ClassLoader classLoader) throws IOException { + M m, Class entityClass, ClassLoader classLoader, Namespace namespace) throws IOException { ProtoSerDe protoSerDe = getProtoSerde(entityClass, classLoader); - return protoSerDe.deserialize(m); + return protoSerDe.deserialize(m, namespace); } private Class loadClass(String className, ClassLoader classLoader) throws IOException { diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java index 324f6a76321..33e44b60fa4 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoSerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.google.protobuf.Message; /** @@ -26,7 +27,8 @@ public interface ProtoSerDe { * Deserializes the provided Protocol Buffer message into its corresponding entity representation. * * @param p The Protocol Buffer message to be deserialized. + * @param namespace The namespace to be specified for entity deserialization. * @return The entity representing the deserialized Protocol Buffer message. */ - T deserialize(M p); + T deserialize(M p, Namespace namespace); } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java index e4a96ae7e63..841a8f9c15e 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.meta.RoleEntity; @@ -44,17 +45,18 @@ public Role serialize(RoleEntity roleEntity) { * @return The entity representing the deserialized Protocol Buffer message. */ @Override - public RoleEntity deserialize(Role role) { + public RoleEntity deserialize(Role role, Namespace namespace) { RoleEntity.Builder builder = RoleEntity.builder() .withId(role.getId()) .withName(role.getName()) + .withNamespace(namespace) .withPrivileges( role.getPrivilegesList().stream() .map(Privileges::fromString) .collect(Collectors.toList())) .withSecurableObject(SecurableObjects.parse(role.getSecurableObject())) - .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo())); + .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); if (!role.getPropertiesMap().isEmpty()) { builder.withProperties(role.getPropertiesMap()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java index 8bef65834d1..1df134bbd4e 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/SchemaEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.SchemaEntity; public class SchemaEntitySerDe implements ProtoSerDe { @@ -27,12 +28,13 @@ public Schema serialize(SchemaEntity schemaEntity) { } @Override - public SchemaEntity deserialize(Schema p) { + public SchemaEntity deserialize(Schema p, Namespace namespace) { SchemaEntity.Builder builder = SchemaEntity.builder() .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java index 06999f8fa8d..b2d79c4f7c9 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/TableEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.TableEntity; public class TableEntitySerDe implements ProtoSerDe { @@ -17,11 +18,12 @@ public Table serialize(TableEntity tableEntity) { } @Override - public TableEntity deserialize(Table p) { + public TableEntity deserialize(Table p, Namespace namespace) { return TableEntity.builder() .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())) + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)) .build(); } } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java index c784030dc47..1fb6ce92275 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/TopicEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.TopicEntity; public class TopicEntitySerDe implements ProtoSerDe { @@ -28,12 +29,13 @@ public Topic serialize(TopicEntity topicEntity) { } @Override - public TopicEntity deserialize(Topic p) { + public TopicEntity deserialize(Topic p, Namespace namespace) { TopicEntity.Builder builder = TopicEntity.builder() .withId(p.getId()) .withName(p.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(p.getAuditInfo(), namespace)); if (p.hasComment()) { builder.withComment(p.getComment()); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java index e87f3314361..47e7f2bb18b 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/UserEntitySerDe.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.meta.UserEntity; import java.util.Collection; @@ -25,12 +26,13 @@ public User serialize(UserEntity userEntity) { } @Override - public UserEntity deserialize(User user) { + public UserEntity deserialize(User user, Namespace namespace) { UserEntity.Builder builder = UserEntity.builder() .withId(user.getId()) .withName(user.getName()) - .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo())); + .withNamespace(namespace) + .withAuditInfo(new AuditInfoSerDe().deserialize(user.getAuditInfo(), namespace)); if (user.getRolesCount() > 0) { builder.withRoles(user.getRolesList()); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 0fafc0c1745..917558888ab 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -135,7 +135,7 @@ public List list( .limit(Integer.MAX_VALUE) .build())); for (Pair pairs : kvs) { - entities.add(serDe.deserialize(pairs.getRight(), e)); + entities.add(serDe.deserialize(pairs.getRight(), e, namespace)); } // TODO (yuqi), if the list is too large, we need to do pagination or streaming return entities; @@ -177,7 +177,7 @@ public E update( throw new NoSuchEntityException(NO_SUCH_ENTITY_MSG, ident.toString()); } - E e = serDe.deserialize(value, type); + E e = serDe.deserialize(value, type, ident.namespace()); E updatedE = updater.apply(e); if (updatedE.nameIdentifier().equals(ident)) { transactionalKvBackend.put(key, serDe.serialize(updatedE), true); @@ -219,7 +219,7 @@ public E get( if (value == null) { throw new NoSuchEntityException(NO_SUCH_ENTITY_MSG, ident.toString()); } - return serDe.deserialize(value, e); + return serDe.deserialize(value, e, ident.namespace()); } void deleteAuthorizationEntitiesIfNecessary(NameIdentifier ident, EntityType type) diff --git a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java index 8223ac239c9..d0b91b79152 100644 --- a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java @@ -4,8 +4,10 @@ */ package com.datastrato.gravitino.proto; +import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.EntitySerDe; import com.datastrato.gravitino.EntitySerDeFactory; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Privileges; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.meta.GroupEntity; @@ -42,7 +44,8 @@ public void testAuditInfoSerDe() throws IOException { byte[] bytes = protoEntitySerDe.serialize(auditInfo); com.datastrato.gravitino.meta.AuditInfo auditInfoFromBytes = - protoEntitySerDe.deserialize(bytes, com.datastrato.gravitino.meta.AuditInfo.class); + protoEntitySerDe.deserialize( + bytes, com.datastrato.gravitino.meta.AuditInfo.class, Namespace.empty()); Assertions.assertEquals(auditInfo, auditInfoFromBytes); // Test with optional fields @@ -55,7 +58,8 @@ public void testAuditInfoSerDe() throws IOException { // Test from/to bytes bytes = protoEntitySerDe.serialize(auditInfo1); auditInfoFromBytes = - protoEntitySerDe.deserialize(bytes, com.datastrato.gravitino.meta.AuditInfo.class); + protoEntitySerDe.deserialize( + bytes, com.datastrato.gravitino.meta.AuditInfo.class, Namespace.empty()); Assertions.assertEquals(auditInfo1, auditInfoFromBytes); // Test with empty field @@ -64,7 +68,8 @@ public void testAuditInfoSerDe() throws IOException { byte[] bytes1 = protoEntitySerDe.serialize(auditInfo2); com.datastrato.gravitino.meta.AuditInfo auditInfoFromBytes1 = - protoEntitySerDe.deserialize(bytes1, com.datastrato.gravitino.meta.AuditInfo.class); + protoEntitySerDe.deserialize( + bytes1, com.datastrato.gravitino.meta.AuditInfo.class, Namespace.empty()); Assertions.assertEquals(auditInfo2, auditInfoFromBytes1); } @@ -98,7 +103,7 @@ public void testEntitiesSerDe() throws IOException { byte[] metalakeBytes = protoEntitySerDe.serialize(metalake); com.datastrato.gravitino.meta.BaseMetalake metalakeFromBytes = protoEntitySerDe.deserialize( - metalakeBytes, com.datastrato.gravitino.meta.BaseMetalake.class); + metalakeBytes, com.datastrato.gravitino.meta.BaseMetalake.class, Namespace.empty()); Assertions.assertEquals(metalake, metalakeFromBytes); // Test metalake without props map @@ -113,7 +118,7 @@ public void testEntitiesSerDe() throws IOException { byte[] metalakeBytes1 = protoEntitySerDe.serialize(metalake1); com.datastrato.gravitino.meta.BaseMetalake metalakeFromBytes1 = protoEntitySerDe.deserialize( - metalakeBytes1, com.datastrato.gravitino.meta.BaseMetalake.class); + metalakeBytes1, com.datastrato.gravitino.meta.BaseMetalake.class, Namespace.empty()); Assertions.assertEquals(metalake1, metalakeFromBytes1); // Test CatalogEntity @@ -121,11 +126,13 @@ public void testEntitiesSerDe() throws IOException { String catalogName = "catalog"; String comment = "comment"; String provider = "test"; + Namespace catalogNamespace = Namespace.of("metalake"); com.datastrato.gravitino.meta.CatalogEntity catalogEntity = com.datastrato.gravitino.meta.CatalogEntity.builder() .withId(catalogId) .withName(catalogName) + .withNamespace(catalogNamespace) .withComment(comment) .withType(com.datastrato.gravitino.Catalog.Type.RELATIONAL) .withProvider(provider) @@ -135,7 +142,7 @@ public void testEntitiesSerDe() throws IOException { byte[] catalogBytes = protoEntitySerDe.serialize(catalogEntity); com.datastrato.gravitino.meta.CatalogEntity catalogEntityFromBytes = protoEntitySerDe.deserialize( - catalogBytes, com.datastrato.gravitino.meta.CatalogEntity.class); + catalogBytes, com.datastrato.gravitino.meta.CatalogEntity.class, catalogNamespace); Assertions.assertEquals(catalogEntity, catalogEntityFromBytes); // Test Fileset catalog @@ -143,6 +150,7 @@ public void testEntitiesSerDe() throws IOException { com.datastrato.gravitino.meta.CatalogEntity.builder() .withId(catalogId) .withName(catalogName) + .withNamespace(catalogNamespace) .withComment(comment) .withType(com.datastrato.gravitino.Catalog.Type.FILESET) .withProvider(provider) @@ -151,22 +159,27 @@ public void testEntitiesSerDe() throws IOException { byte[] filesetCatalogBytes = protoEntitySerDe.serialize(filesetCatalogEntity); com.datastrato.gravitino.meta.CatalogEntity filesetCatalogEntityFromBytes = protoEntitySerDe.deserialize( - filesetCatalogBytes, com.datastrato.gravitino.meta.CatalogEntity.class); + filesetCatalogBytes, + com.datastrato.gravitino.meta.CatalogEntity.class, + catalogNamespace); Assertions.assertEquals(filesetCatalogEntity, filesetCatalogEntityFromBytes); // Test SchemaEntity + Namespace schemaNamespace = Namespace.of("metalake", "catalog"); Long schemaId = 1L; String schemaName = "schema"; com.datastrato.gravitino.meta.SchemaEntity schemaEntity = com.datastrato.gravitino.meta.SchemaEntity.builder() .withId(schemaId) .withName(schemaName) + .withNamespace(schemaNamespace) .withAuditInfo(auditInfo) .build(); byte[] schemaBytes = protoEntitySerDe.serialize(schemaEntity); com.datastrato.gravitino.meta.SchemaEntity schemaEntityFromBytes = - protoEntitySerDe.deserialize(schemaBytes, com.datastrato.gravitino.meta.SchemaEntity.class); + protoEntitySerDe.deserialize( + schemaBytes, com.datastrato.gravitino.meta.SchemaEntity.class, schemaNamespace); Assertions.assertEquals(schemaEntity, schemaEntityFromBytes); // Test SchemaEntity with additional fields @@ -174,6 +187,7 @@ public void testEntitiesSerDe() throws IOException { com.datastrato.gravitino.meta.SchemaEntity.builder() .withId(schemaId) .withName(schemaName) + .withNamespace(schemaNamespace) .withAuditInfo(auditInfo) .withComment(comment) .withProperties(props) @@ -181,33 +195,38 @@ public void testEntitiesSerDe() throws IOException { byte[] schemaBytes1 = protoEntitySerDe.serialize(schemaEntity1); com.datastrato.gravitino.meta.SchemaEntity schemaEntityFromBytes1 = protoEntitySerDe.deserialize( - schemaBytes1, com.datastrato.gravitino.meta.SchemaEntity.class); + schemaBytes1, com.datastrato.gravitino.meta.SchemaEntity.class, schemaNamespace); Assertions.assertEquals(schemaEntity1, schemaEntityFromBytes1); Assertions.assertEquals(comment, schemaEntityFromBytes1.comment()); Assertions.assertEquals(props, schemaEntityFromBytes1.properties()); // Test TableEntity + Namespace tableNamespace = Namespace.of("metalake", "catalog", "schema"); Long tableId = 1L; String tableName = "table"; com.datastrato.gravitino.meta.TableEntity tableEntity = com.datastrato.gravitino.meta.TableEntity.builder() .withId(tableId) .withName(tableName) + .withNamespace(tableNamespace) .withAuditInfo(auditInfo) .build(); byte[] tableBytes = protoEntitySerDe.serialize(tableEntity); com.datastrato.gravitino.meta.TableEntity tableEntityFromBytes = - protoEntitySerDe.deserialize(tableBytes, com.datastrato.gravitino.meta.TableEntity.class); + protoEntitySerDe.deserialize( + tableBytes, com.datastrato.gravitino.meta.TableEntity.class, tableNamespace); Assertions.assertEquals(tableEntity, tableEntityFromBytes); // Test FileEntity + Namespace filesetNamespace = Namespace.of("metalake", "catalog", "schema"); Long fileId = 1L; String fileName = "file"; com.datastrato.gravitino.meta.FilesetEntity fileEntity = com.datastrato.gravitino.meta.FilesetEntity.builder() .withId(fileId) .withName(fileName) + .withNamespace(filesetNamespace) .withAuditInfo(auditInfo) .withFilesetType(com.datastrato.gravitino.file.Fileset.Type.MANAGED) .withStorageLocation("testLocation") @@ -216,20 +235,23 @@ public void testEntitiesSerDe() throws IOException { .build(); byte[] fileBytes = protoEntitySerDe.serialize(fileEntity); com.datastrato.gravitino.meta.FilesetEntity fileEntityFromBytes = - protoEntitySerDe.deserialize(fileBytes, com.datastrato.gravitino.meta.FilesetEntity.class); + protoEntitySerDe.deserialize( + fileBytes, com.datastrato.gravitino.meta.FilesetEntity.class, filesetNamespace); Assertions.assertEquals(fileEntity, fileEntityFromBytes); com.datastrato.gravitino.meta.FilesetEntity fileEntity1 = com.datastrato.gravitino.meta.FilesetEntity.builder() .withId(fileId) .withName(fileName) + .withNamespace(filesetNamespace) .withAuditInfo(auditInfo) .withFilesetType(com.datastrato.gravitino.file.Fileset.Type.MANAGED) .withStorageLocation("testLocation") .build(); byte[] fileBytes1 = protoEntitySerDe.serialize(fileEntity1); com.datastrato.gravitino.meta.FilesetEntity fileEntityFromBytes1 = - protoEntitySerDe.deserialize(fileBytes1, com.datastrato.gravitino.meta.FilesetEntity.class); + protoEntitySerDe.deserialize( + fileBytes1, com.datastrato.gravitino.meta.FilesetEntity.class, filesetNamespace); Assertions.assertEquals(fileEntity1, fileEntityFromBytes1); Assertions.assertNull(fileEntityFromBytes1.comment()); Assertions.assertNull(fileEntityFromBytes1.properties()); @@ -238,6 +260,7 @@ public void testEntitiesSerDe() throws IOException { com.datastrato.gravitino.meta.FilesetEntity.builder() .withId(fileId) .withName(fileName) + .withNamespace(filesetNamespace) .withAuditInfo(auditInfo) .withFilesetType(com.datastrato.gravitino.file.Fileset.Type.EXTERNAL) .withProperties(props) @@ -246,63 +269,80 @@ public void testEntitiesSerDe() throws IOException { .build(); byte[] fileBytes2 = protoEntitySerDe.serialize(fileEntity2); com.datastrato.gravitino.meta.FilesetEntity fileEntityFromBytes2 = - protoEntitySerDe.deserialize(fileBytes2, com.datastrato.gravitino.meta.FilesetEntity.class); + protoEntitySerDe.deserialize( + fileBytes2, com.datastrato.gravitino.meta.FilesetEntity.class, filesetNamespace); Assertions.assertEquals(fileEntity2, fileEntityFromBytes2); Assertions.assertEquals("testLocation", fileEntityFromBytes2.storageLocation()); Assertions.assertEquals( com.datastrato.gravitino.file.Fileset.Type.EXTERNAL, fileEntityFromBytes2.filesetType()); // Test TopicEntity + Namespace topicNamespace = Namespace.of("metalake", "catalog", "default"); Long topicId = 1L; String topicName = "topic"; com.datastrato.gravitino.meta.TopicEntity topicEntity = com.datastrato.gravitino.meta.TopicEntity.builder() .withId(topicId) .withName(topicName) + .withNamespace(topicNamespace) .withAuditInfo(auditInfo) .withComment(comment) .withProperties(props) .build(); byte[] topicBytes = protoEntitySerDe.serialize(topicEntity); com.datastrato.gravitino.meta.TopicEntity topicEntityFromBytes = - protoEntitySerDe.deserialize(topicBytes, com.datastrato.gravitino.meta.TopicEntity.class); + protoEntitySerDe.deserialize( + topicBytes, com.datastrato.gravitino.meta.TopicEntity.class, topicNamespace); Assertions.assertEquals(topicEntity, topicEntityFromBytes); com.datastrato.gravitino.meta.TopicEntity topicEntity1 = com.datastrato.gravitino.meta.TopicEntity.builder() .withId(topicId) .withName(topicName) + .withNamespace(topicNamespace) .withAuditInfo(auditInfo) .build(); byte[] topicBytes1 = protoEntitySerDe.serialize(topicEntity1); com.datastrato.gravitino.meta.TopicEntity topicEntityFromBytes1 = - protoEntitySerDe.deserialize(topicBytes1, com.datastrato.gravitino.meta.TopicEntity.class); + protoEntitySerDe.deserialize( + topicBytes1, com.datastrato.gravitino.meta.TopicEntity.class, topicNamespace); Assertions.assertEquals(topicEntity1, topicEntityFromBytes1); Assertions.assertNull(topicEntityFromBytes1.comment()); Assertions.assertNull(topicEntityFromBytes1.properties()); // Test UserEntity + Namespace userNamespace = + Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.USER_SCHEMA_NAME); Long userId = 1L; String userName = "user"; UserEntity userEntity = UserEntity.builder() .withId(userId) .withName(userName) + .withNamespace(userNamespace) .withAuditInfo(auditInfo) .withRoles(Lists.newArrayList("role")) .build(); byte[] userBytes = protoEntitySerDe.serialize(userEntity); - UserEntity userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class); + UserEntity userEntityFromBytes = + protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); Assertions.assertEquals(userEntity, userEntityFromBytes); UserEntity userEntityWithoutFields = - UserEntity.builder().withId(userId).withName(userName).withAuditInfo(auditInfo).build(); + UserEntity.builder() + .withId(userId) + .withName(userName) + .withNamespace(userNamespace) + .withAuditInfo(auditInfo) + .build(); userBytes = protoEntitySerDe.serialize(userEntityWithoutFields); - userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class); + userEntityFromBytes = protoEntitySerDe.deserialize(userBytes, UserEntity.class, userNamespace); Assertions.assertEquals(userEntityWithoutFields, userEntityFromBytes); Assertions.assertNull(userEntityWithoutFields.roles()); // Test GroupEntity + Namespace groupNamespace = + Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.GROUP_SCHEMA_NAME); Long groupId = 1L; String groupName = "group"; @@ -310,46 +350,58 @@ public void testEntitiesSerDe() throws IOException { GroupEntity.builder() .withId(groupId) .withName(groupName) + .withNamespace(groupNamespace) .withAuditInfo(auditInfo) .withRoles(Lists.newArrayList("role")) .build(); byte[] groupBytes = protoEntitySerDe.serialize(group); - GroupEntity groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class); + GroupEntity groupFromBytes = + protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); Assertions.assertEquals(group, groupFromBytes); GroupEntity groupWithoutFields = - GroupEntity.builder().withId(groupId).withName(groupName).withAuditInfo(auditInfo).build(); + GroupEntity.builder() + .withId(groupId) + .withName(groupName) + .withNamespace(groupNamespace) + .withAuditInfo(auditInfo) + .build(); groupBytes = protoEntitySerDe.serialize(groupWithoutFields); - groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class); + groupFromBytes = protoEntitySerDe.deserialize(groupBytes, GroupEntity.class, groupNamespace); Assertions.assertEquals(groupWithoutFields, groupFromBytes); Assertions.assertNull(groupWithoutFields.roles()); // Test RoleEntity + Namespace roleNamespace = + Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME); Long roleId = 1L; String roleName = "testRole"; RoleEntity roleEntity = RoleEntity.builder() .withId(roleId) .withName(roleName) + .withNamespace(roleNamespace) .withAuditInfo(auditInfo) .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .withProperties(props) .build(); byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); - RoleEntity roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class); + RoleEntity roleFromBytes = + protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); Assertions.assertEquals(roleEntity, roleFromBytes); RoleEntity roleWithoutFields = RoleEntity.builder() .withId(1L) .withName(roleName) + .withNamespace(roleNamespace) .withAuditInfo(auditInfo) .withSecurableObject(SecurableObjects.of(catalogName)) .withPrivileges(Lists.newArrayList(Privileges.LoadCatalog.get())) .build(); roleBytes = protoEntitySerDe.serialize(roleWithoutFields); - roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class); + roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); Assertions.assertEquals(roleWithoutFields, roleFromBytes); } }