From db13491570d69afdd625c5d92607cccc5a15d873 Mon Sep 17 00:00:00 2001 From: "xiang.oyx" Date: Thu, 7 Nov 2024 17:51:41 +0800 Subject: [PATCH 1/3] add deserial object with feature deserializeNonexistentClassNotWriteFullClassInfo --- .../java/org/apache/fury/config/Config.java | 12 ++++++ .../org/apache/fury/config/FuryBuilder.java | 16 ++++++++ .../NonexistentClassSerializers.java | 20 ++++++++-- .../NonexistentClassSerializersTest.java | 38 +++++++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/config/Config.java b/java/fury-core/src/main/java/org/apache/fury/config/Config.java index 98301413b3..12c7488299 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/Config.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/Config.java @@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; + import org.apache.fury.Fury; import org.apache.fury.meta.MetaCompressor; import org.apache.fury.serializer.Serializer; @@ -57,6 +58,7 @@ public class Config implements Serializable { private final MetaCompressor metaCompressor; private final boolean asyncCompilationEnabled; private final boolean deserializeNonexistentClass; + private final boolean deserializeNonexistentClassNotWriteFullClassInfo; private final boolean scalaOptimizationEnabled; private transient int configHash; private final boolean deserializeNonexistentEnumValueAsNull; @@ -91,6 +93,7 @@ public Config(FuryBuilder builder) { // unexisted class by type info in data. Preconditions.checkArgument(metaShareEnabled || compatibleMode == CompatibleMode.COMPATIBLE); } + deserializeNonexistentClassNotWriteFullClassInfo = builder.deserializeNonexistentClassNotWriteFullClassInfo; asyncCompilationEnabled = builder.asyncCompilationEnabled; scalaOptimizationEnabled = builder.scalaOptimizationEnabled; deserializeNonexistentEnumValueAsNull = builder.deserializeNonexistentEnumValueAsNull; @@ -239,6 +242,13 @@ public boolean deserializeNonexistentClass() { return deserializeNonexistentClass; } + /** + * Whether deserialize/skip data of un-existed class with full Class info. if enable then not write full class info + */ + public boolean deserializeNonexistentClassNotWriteFullClassInfo() { + return deserializeNonexistentClassNotWriteFullClassInfo; + } + /** * Whether JIT is enabled. * @@ -291,6 +301,7 @@ public boolean equals(Object o) { && Objects.equals(metaCompressor, config.metaCompressor) && asyncCompilationEnabled == config.asyncCompilationEnabled && deserializeNonexistentClass == config.deserializeNonexistentClass + && deserializeNonexistentClassNotWriteFullClassInfo == config.deserializeNonexistentClassNotWriteFullClassInfo && scalaOptimizationEnabled == config.scalaOptimizationEnabled && language == config.language && compatibleMode == config.compatibleMode @@ -325,6 +336,7 @@ public int hashCode() { metaCompressor, asyncCompilationEnabled, deserializeNonexistentClass, + deserializeNonexistentClassNotWriteFullClassInfo, scalaOptimizationEnabled); } diff --git a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java index 8e183d66fb..6028498809 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java @@ -77,6 +77,12 @@ public final class FuryBuilder { Boolean scopedMetaShareEnabled; boolean codeGenEnabled = true; Boolean deserializeNonexistentClass; + /** + * Whether deserialize nonexistent class not write full class info + * default write full class info + * if enable then not write full class info + */ + boolean deserializeNonexistentClassNotWriteFullClassInfo = false; boolean asyncCompilationEnabled = false; boolean registerGuavaTypes = true; boolean scalaOptimizationEnabled = false; @@ -297,6 +303,16 @@ public FuryBuilder withDeserializeNonexistentClass(boolean deserializeNonexisten return this; } + /** + * Whether deserialize/skip data of un-existed class. if write class full info + * + * @see Config#deserializeNonexistentClassNotWriteFullClassInfo() + */ + public FuryBuilder withDeserializeNonexistentClassNotWriteFullClassInfo(boolean deserializeNonexistentClassNotWriteFullClassInfo) { + this.deserializeNonexistentClassNotWriteFullClassInfo = deserializeNonexistentClassNotWriteFullClassInfo; + return this; + } + /** * Whether enable jit for serialization. When disabled, the first serialization will be faster * since no need to generate code, but later will be much slower compared jit mode. diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java index bbdd972e65..e51064b1c0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java @@ -28,6 +28,7 @@ import org.apache.fury.collection.MapEntry; import org.apache.fury.collection.Tuple2; import org.apache.fury.collection.Tuple3; +import org.apache.fury.config.Config; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.meta.ClassDef; import org.apache.fury.resolver.ClassInfo; @@ -193,6 +194,7 @@ public Object read(MemoryBuffer buffer) { ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); ObjectSerializer.FinalTypeField[] finalFields = fieldsInfo.finalFields; boolean[] isFinal = fieldsInfo.isFinal; + Config config = fury.getConfig(); for (int i = 0; i < finalFields.length; i++) { ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; Object fieldValue; @@ -208,23 +210,35 @@ public Object read(MemoryBuffer buffer) { fury, refResolver, classResolver, fieldInfo, isFinal[i], buffer); } } - entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); + entries.add(new MapEntry(getFileName(fieldInfo.qualifiedFieldName, config), fieldValue)); } for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { Object fieldValue = ObjectSerializer.readOtherFieldValue(fury, fieldInfo, buffer); - entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); + entries.add(new MapEntry(getFileName(fieldInfo.qualifiedFieldName, config), fieldValue)); } Generics generics = fury.getGenerics(); for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) { Object fieldValue = ObjectSerializer.readContainerFieldValue(fury, generics, fieldInfo, buffer); - entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); + entries.add(new MapEntry(getFileName(fieldInfo.qualifiedFieldName, config), fieldValue)); } obj.setEntries(entries); return obj; } } + public static String getFileName(String qualifiedFieldName, Config config) { + if (config.deserializeNonexistentClassNotWriteFullClassInfo()) { + int index = qualifiedFieldName.lastIndexOf("."); + if (index < 0) { + return qualifiedFieldName; + } + return qualifiedFieldName.substring(index + 1); + } else { + return qualifiedFieldName; + } + } + public static final class NonexistentEnumClassSerializer extends Serializer { private final NonexistentEnum[] enumConstants; private final MetaStringResolver metaStringResolver; diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java index 05a5dcdc23..af986f7d0c 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java @@ -108,6 +108,44 @@ public void testSkipNonexistent( } } + @Test(dataProvider = "config") + public void testSkipNonexistentWithNoneClassInfo( + boolean referenceTracking, + boolean scopedMetaShare, + boolean enableCodegen1, + boolean enableCodegen2) { + Fury fury = + furyBuilder(scopedMetaShare) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen1) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withDeserializeNonexistentClassNotWriteFullClassInfo(true) + .build(); + ClassLoader classLoader = getClass().getClassLoader(); + for (Class structClass : + new Class[] { + Struct.createNumberStructClass("TestSkipNonexistentClass1", 2), + Struct.createStructClass("TestSkipNonexistentClass1", 2) + }) { + Object pojo = Struct.createPOJO(structClass); + byte[] bytes = fury.serialize(pojo); + Fury fury2 = + furyBuilder(scopedMetaShare) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen2) + .withClassLoader(classLoader) + .withDeserializeNonexistentClassNotWriteFullClassInfo(true) + .build(); + Object o = fury2.deserialize(bytes); + if (o instanceof NonexistentClass.NonexistentMetaShared) { + System.out.println(o); + NonexistentClass.NonexistentMetaShared s = (NonexistentClass.NonexistentMetaShared)o; + Assert.assertNotNull(s.get("f0")); + } + assertTrue(o instanceof NonexistentClass, "Unexpect type " + o.getClass()); + } + } + @Test(dataProvider = "scopedMetaShare") public void testNonexistentEnum(boolean scopedMetaShare) { Fury fury = furyBuilder(scopedMetaShare).withDeserializeNonexistentClass(true).build(); From fe9908984d69005faaabb20ff772bcf7af2a87af Mon Sep 17 00:00:00 2001 From: "xiang.oyx" Date: Thu, 7 Nov 2024 20:22:06 +0800 Subject: [PATCH 2/3] add deserial object with feature deserializeNonexistentClassNotWriteFullClassInfo --- .../src/main/java/org/apache/fury/config/FuryBuilder.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java index 6028498809..a05b69ec63 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java @@ -77,11 +77,6 @@ public final class FuryBuilder { Boolean scopedMetaShareEnabled; boolean codeGenEnabled = true; Boolean deserializeNonexistentClass; - /** - * Whether deserialize nonexistent class not write full class info - * default write full class info - * if enable then not write full class info - */ boolean deserializeNonexistentClassNotWriteFullClassInfo = false; boolean asyncCompilationEnabled = false; boolean registerGuavaTypes = true; From 96ca7ca1fb07b60d19f0f421b989dc4a20fbc8d4 Mon Sep 17 00:00:00 2001 From: "xiang.oyx" Date: Thu, 7 Nov 2024 22:31:27 +0800 Subject: [PATCH 3/3] add deserial object with feature deserializeNonexistentClassNotWriteFullClassInfo --- .../java/org/apache/fury/config/Config.java | 10 +-- .../org/apache/fury/config/FuryBuilder.java | 6 +- .../NonexistentClassSerializers.java | 16 ++--- .../NonexistentClassSerializersTest.java | 72 +++++++++---------- 4 files changed, 54 insertions(+), 50 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/config/Config.java b/java/fury-core/src/main/java/org/apache/fury/config/Config.java index 12c7488299..f061a3ee0a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/Config.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/Config.java @@ -24,7 +24,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.fury.Fury; import org.apache.fury.meta.MetaCompressor; import org.apache.fury.serializer.Serializer; @@ -93,7 +92,8 @@ public Config(FuryBuilder builder) { // unexisted class by type info in data. Preconditions.checkArgument(metaShareEnabled || compatibleMode == CompatibleMode.COMPATIBLE); } - deserializeNonexistentClassNotWriteFullClassInfo = builder.deserializeNonexistentClassNotWriteFullClassInfo; + deserializeNonexistentClassNotWriteFullClassInfo = + builder.deserializeNonexistentClassNotWriteFullClassInfo; asyncCompilationEnabled = builder.asyncCompilationEnabled; scalaOptimizationEnabled = builder.scalaOptimizationEnabled; deserializeNonexistentEnumValueAsNull = builder.deserializeNonexistentEnumValueAsNull; @@ -243,7 +243,8 @@ public boolean deserializeNonexistentClass() { } /** - * Whether deserialize/skip data of un-existed class with full Class info. if enable then not write full class info + * Whether deserialize/skip data of un-existed class with full Class info. if enable then not + * write full class info */ public boolean deserializeNonexistentClassNotWriteFullClassInfo() { return deserializeNonexistentClassNotWriteFullClassInfo; @@ -301,7 +302,8 @@ public boolean equals(Object o) { && Objects.equals(metaCompressor, config.metaCompressor) && asyncCompilationEnabled == config.asyncCompilationEnabled && deserializeNonexistentClass == config.deserializeNonexistentClass - && deserializeNonexistentClassNotWriteFullClassInfo == config.deserializeNonexistentClassNotWriteFullClassInfo + && deserializeNonexistentClassNotWriteFullClassInfo + == config.deserializeNonexistentClassNotWriteFullClassInfo && scalaOptimizationEnabled == config.scalaOptimizationEnabled && language == config.language && compatibleMode == config.compatibleMode diff --git a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java index a05b69ec63..b97e7fdeea 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java @@ -303,8 +303,10 @@ public FuryBuilder withDeserializeNonexistentClass(boolean deserializeNonexisten * * @see Config#deserializeNonexistentClassNotWriteFullClassInfo() */ - public FuryBuilder withDeserializeNonexistentClassNotWriteFullClassInfo(boolean deserializeNonexistentClassNotWriteFullClassInfo) { - this.deserializeNonexistentClassNotWriteFullClassInfo = deserializeNonexistentClassNotWriteFullClassInfo; + public FuryBuilder withDeserializeNonexistentClassNotWriteFullClassInfo( + boolean deserializeNonexistentClassNotWriteFullClassInfo) { + this.deserializeNonexistentClassNotWriteFullClassInfo = + deserializeNonexistentClassNotWriteFullClassInfo; return this; } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java index e51064b1c0..6a17197d8e 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java @@ -228,15 +228,15 @@ public Object read(MemoryBuffer buffer) { } public static String getFileName(String qualifiedFieldName, Config config) { - if (config.deserializeNonexistentClassNotWriteFullClassInfo()) { - int index = qualifiedFieldName.lastIndexOf("."); - if (index < 0) { - return qualifiedFieldName; - } - return qualifiedFieldName.substring(index + 1); - } else { - return qualifiedFieldName; + if (config.deserializeNonexistentClassNotWriteFullClassInfo()) { + int index = qualifiedFieldName.lastIndexOf("."); + if (index < 0) { + return qualifiedFieldName; } + return qualifiedFieldName.substring(index + 1); + } else { + return qualifiedFieldName; + } } public static final class NonexistentEnumClassSerializer extends Serializer { diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java index af986f7d0c..9297e108ea 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java @@ -108,43 +108,43 @@ public void testSkipNonexistent( } } - @Test(dataProvider = "config") - public void testSkipNonexistentWithNoneClassInfo( - boolean referenceTracking, - boolean scopedMetaShare, - boolean enableCodegen1, - boolean enableCodegen2) { - Fury fury = - furyBuilder(scopedMetaShare) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen1) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withDeserializeNonexistentClassNotWriteFullClassInfo(true) - .build(); - ClassLoader classLoader = getClass().getClassLoader(); - for (Class structClass : - new Class[] { - Struct.createNumberStructClass("TestSkipNonexistentClass1", 2), - Struct.createStructClass("TestSkipNonexistentClass1", 2) - }) { - Object pojo = Struct.createPOJO(structClass); - byte[] bytes = fury.serialize(pojo); - Fury fury2 = - furyBuilder(scopedMetaShare) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen2) - .withClassLoader(classLoader) - .withDeserializeNonexistentClassNotWriteFullClassInfo(true) - .build(); - Object o = fury2.deserialize(bytes); - if (o instanceof NonexistentClass.NonexistentMetaShared) { - System.out.println(o); - NonexistentClass.NonexistentMetaShared s = (NonexistentClass.NonexistentMetaShared)o; - Assert.assertNotNull(s.get("f0")); - } - assertTrue(o instanceof NonexistentClass, "Unexpect type " + o.getClass()); - } + @Test(dataProvider = "config") + public void testSkipNonexistentWithNoneClassInfo( + boolean referenceTracking, + boolean scopedMetaShare, + boolean enableCodegen1, + boolean enableCodegen2) { + Fury fury = + furyBuilder(scopedMetaShare) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen1) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withDeserializeNonexistentClassNotWriteFullClassInfo(true) + .build(); + ClassLoader classLoader = getClass().getClassLoader(); + for (Class structClass : + new Class[] { + Struct.createNumberStructClass("TestSkipNonexistentClass1", 2), + Struct.createStructClass("TestSkipNonexistentClass1", 2) + }) { + Object pojo = Struct.createPOJO(structClass); + byte[] bytes = fury.serialize(pojo); + Fury fury2 = + furyBuilder(scopedMetaShare) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen2) + .withClassLoader(classLoader) + .withDeserializeNonexistentClassNotWriteFullClassInfo(true) + .build(); + Object o = fury2.deserialize(bytes); + if (o instanceof NonexistentClass.NonexistentMetaShared) { + System.out.println(o); + NonexistentClass.NonexistentMetaShared s = (NonexistentClass.NonexistentMetaShared) o; + Assert.assertNotNull(s.get("f0")); + } + assertTrue(o instanceof NonexistentClass, "Unexpect type " + o.getClass()); } + } @Test(dataProvider = "scopedMetaShare") public void testNonexistentEnum(boolean scopedMetaShare) {