Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(java): add deserial object with feature deserializeNonexistentClassNotWriteFullClassInfo feature #1932

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat(java): add deserial object with feature deserializeNonexistentClassNotWriteFullClassInfo feature #1932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
db13491
add deserial object with feature deserializeNonexistentClassNotWriteF…
Nov 7, 2024
fe99089
add deserial object with feature deserializeNonexistentClassNotWriteF…
Nov 7, 2024
96ca7ca
add deserial object with feature deserializeNonexistentClassNotWriteF…
Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions java/fury-core/src/main/java/org/apache/fury/config/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public class Config implements Serializable {
private final MetaCompressor metaCompressor;
private final boolean asyncCompilationEnabled;
private final boolean deserializeNonexistentClass;
private final boolean deserializeNonexistentClassNotWriteFullClassInfo;
private final boolean scalaOptimizationEnabled;
private transient int configHash;
private final boolean deserializeNonexistentEnumValueAsNull;
Expand Down Expand Up @@ -91,6 +92,8 @@ public Config(FuryBuilder builder) {
// unexisted class by type info in data.
Preconditions.checkArgument(metaShareEnabled || compatibleMode == CompatibleMode.COMPATIBLE);
}
deserializeNonexistentClassNotWriteFullClassInfo =
builder.deserializeNonexistentClassNotWriteFullClassInfo;
asyncCompilationEnabled = builder.asyncCompilationEnabled;
scalaOptimizationEnabled = builder.scalaOptimizationEnabled;
deserializeNonexistentEnumValueAsNull = builder.deserializeNonexistentEnumValueAsNull;
Expand Down Expand Up @@ -239,6 +242,14 @@ public boolean deserializeNonexistentClass() {
return deserializeNonexistentClass;
}

/**
* Whether deserialize/skip data of un-existed class with full Class info. if enable then not
* write full class info
*/
public boolean deserializeNonexistentClassNotWriteFullClassInfo() {
return deserializeNonexistentClassNotWriteFullClassInfo;
}

/**
* Whether JIT is enabled.
*
Expand Down Expand Up @@ -291,6 +302,8 @@ public boolean equals(Object o) {
&& Objects.equals(metaCompressor, config.metaCompressor)
&& asyncCompilationEnabled == config.asyncCompilationEnabled
&& deserializeNonexistentClass == config.deserializeNonexistentClass
&& deserializeNonexistentClassNotWriteFullClassInfo
== config.deserializeNonexistentClassNotWriteFullClassInfo
&& scalaOptimizationEnabled == config.scalaOptimizationEnabled
&& language == config.language
&& compatibleMode == config.compatibleMode
Expand Down Expand Up @@ -325,6 +338,7 @@ public int hashCode() {
metaCompressor,
asyncCompilationEnabled,
deserializeNonexistentClass,
deserializeNonexistentClassNotWriteFullClassInfo,
scalaOptimizationEnabled);
}

Expand Down
13 changes: 13 additions & 0 deletions java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ public final class FuryBuilder {
Boolean scopedMetaShareEnabled;
boolean codeGenEnabled = true;
Boolean deserializeNonexistentClass;
boolean deserializeNonexistentClassNotWriteFullClassInfo = false;
boolean asyncCompilationEnabled = false;
boolean registerGuavaTypes = true;
boolean scalaOptimizationEnabled = false;
Expand Down Expand Up @@ -297,6 +298,18 @@ public FuryBuilder withDeserializeNonexistentClass(boolean deserializeNonexisten
return this;
}

/**
* Whether deserialize/skip data of un-existed class. if write class full info
*
* @see Config#deserializeNonexistentClassNotWriteFullClassInfo()
*/
public FuryBuilder withDeserializeNonexistentClassNotWriteFullClassInfo(
boolean deserializeNonexistentClassNotWriteFullClassInfo) {
this.deserializeNonexistentClassNotWriteFullClassInfo =
deserializeNonexistentClassNotWriteFullClassInfo;
return this;
}

/**
* Whether enable jit for serialization. When disabled, the first serialization will be faster
* since no need to generate code, but later will be much slower compared jit mode.
Expand Down
20 changes: 17 additions & 3 deletions java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.fury.collection.MapEntry;
import org.apache.fury.collection.Tuple2;
import org.apache.fury.collection.Tuple3;
import org.apache.fury.config.Config;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.meta.ClassDef;
import org.apache.fury.resolver.ClassInfo;
Expand Down Expand Up @@ -193,6 +194,7 @@ public Object read(MemoryBuffer buffer) {
ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef);
ObjectSerializer.FinalTypeField[] finalFields = fieldsInfo.finalFields;
boolean[] isFinal = fieldsInfo.isFinal;
Config config = fury.getConfig();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'd better get the option here, and pass that flag to getFileName method

for (int i = 0; i < finalFields.length; i++) {
ObjectSerializer.FinalTypeField fieldInfo = finalFields[i];
Object fieldValue;
Expand All @@ -208,23 +210,35 @@ public Object read(MemoryBuffer buffer) {
fury, refResolver, classResolver, fieldInfo, isFinal[i], buffer);
}
}
entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue));
entries.add(new MapEntry(getFileName(fieldInfo.qualifiedFieldName, config), fieldValue));
}
for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) {
Object fieldValue = ObjectSerializer.readOtherFieldValue(fury, fieldInfo, buffer);
entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue));
entries.add(new MapEntry(getFileName(fieldInfo.qualifiedFieldName, config), fieldValue));
}
Generics generics = fury.getGenerics();
for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) {
Object fieldValue =
ObjectSerializer.readContainerFieldValue(fury, generics, fieldInfo, buffer);
entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue));
entries.add(new MapEntry(getFileName(fieldInfo.qualifiedFieldName, config), fieldValue));
}
obj.setEntries(entries);
return obj;
}
}

public static String getFileName(String qualifiedFieldName, Config config) {
if (config.deserializeNonexistentClassNotWriteFullClassInfo()) {
int index = qualifiedFieldName.lastIndexOf(".");
if (index < 0) {
return qualifiedFieldName;
}
return qualifiedFieldName.substring(index + 1);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will create a new string everytime for deserialization, which is very slow. A better method is rename fieldInfo#qualifiedFieldName into fieldInfo#fieldName, and build it as qualified based on the option.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And we should skip duplicated field name in super class if this option is enabled

} else {
return qualifiedFieldName;
}
}

public static final class NonexistentEnumClassSerializer extends Serializer {
private final NonexistentEnum[] enumConstants;
private final MetaStringResolver metaStringResolver;
Expand Down
38 changes: 38 additions & 0 deletions java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,44 @@ public void testSkipNonexistent(
}
}

@Test(dataProvider = "config")
public void testSkipNonexistentWithNoneClassInfo(
boolean referenceTracking,
boolean scopedMetaShare,
boolean enableCodegen1,
boolean enableCodegen2) {
Fury fury =
furyBuilder(scopedMetaShare)
.withRefTracking(referenceTracking)
.withCodegen(enableCodegen1)
.withCompatibleMode(CompatibleMode.COMPATIBLE)
.withDeserializeNonexistentClassNotWriteFullClassInfo(true)
.build();
ClassLoader classLoader = getClass().getClassLoader();
for (Class<?> structClass :
new Class<?>[] {
Struct.createNumberStructClass("TestSkipNonexistentClass1", 2),
Struct.createStructClass("TestSkipNonexistentClass1", 2)
}) {
Object pojo = Struct.createPOJO(structClass);
byte[] bytes = fury.serialize(pojo);
Fury fury2 =
furyBuilder(scopedMetaShare)
.withRefTracking(referenceTracking)
.withCodegen(enableCodegen2)
.withClassLoader(classLoader)
.withDeserializeNonexistentClassNotWriteFullClassInfo(true)
.build();
Object o = fury2.deserialize(bytes);
if (o instanceof NonexistentClass.NonexistentMetaShared) {
System.out.println(o);
NonexistentClass.NonexistentMetaShared s = (NonexistentClass.NonexistentMetaShared) o;
Assert.assertNotNull(s.get("f0"));
}
assertTrue(o instanceof NonexistentClass, "Unexpect type " + o.getClass());
}
}

@Test(dataProvider = "scopedMetaShare")
public void testNonexistentEnum(boolean scopedMetaShare) {
Fury fury = furyBuilder(scopedMetaShare).withDeserializeNonexistentClass(true).build();
Expand Down
Loading