From 4e0adfc219bb9cef2bd5c03efca10bba595a8463 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Thu, 11 May 2023 10:46:40 +0800 Subject: [PATCH 1/2] support Compatible Serialization --- .../src/main/java/io/fury/Config.java | 7 + .../fury-core/src/main/java/io/fury/Fury.java | 7 + .../java/io/fury/resolver/ClassResolver.java | 17 +- .../java/io/fury/resolver/FieldResolver.java | 1 + .../fury/serializer/CompatibleSerializer.java | 600 ++++++++++++++++++ .../src/test/java/io/fury/FuryTestBase.java | 10 + .../serializer/CompatibleSerializerTest.java | 474 ++++++++++++++ .../src/main/java/io/fury/test/bean/Foo.java | 33 + 8 files changed, 1147 insertions(+), 2 deletions(-) create mode 100644 java/fury-core/src/main/java/io/fury/serializer/CompatibleSerializer.java create mode 100644 java/fury-core/src/test/java/io/fury/serializer/CompatibleSerializerTest.java diff --git a/java/fury-core/src/main/java/io/fury/Config.java b/java/fury-core/src/main/java/io/fury/Config.java index fbbff7c298..4ac7b349a9 100644 --- a/java/fury-core/src/main/java/io/fury/Config.java +++ b/java/fury-core/src/main/java/io/fury/Config.java @@ -36,6 +36,7 @@ package io.fury; +import io.fury.serializer.CompatibleMode; import io.fury.serializer.Serializer; import io.fury.serializer.TimeSerializers; import io.fury.util.MurmurHash3; @@ -58,6 +59,7 @@ public class Config implements Serializable { private final boolean compressNumber; private final boolean compressString; private final boolean checkClassVersion; + private final CompatibleMode compatibleMode; private final Class defaultJDKStreamSerializerType; private final boolean secureModeEnabled; private final boolean classRegistrationRequired; @@ -73,6 +75,7 @@ public class Config implements Serializable { compressNumber = builder.compressNumber; compressString = builder.compressString; checkClassVersion = builder.checkClassVersion; + compatibleMode = builder.compatibleMode; defaultJDKStreamSerializerType = builder.defaultJDKStreamSerializerType; secureModeEnabled = builder.secureModeEnabled; classRegistrationRequired = builder.requireClassRegistration; @@ -125,6 +128,10 @@ public boolean checkClassVersion() { return checkClassVersion; } + public CompatibleMode getCompatibleMode() { + return compatibleMode; + } + /** * Returns default serializer type for class which implements jdk serialization method such as * `writeObject/readObject`. diff --git a/java/fury-core/src/main/java/io/fury/Fury.java b/java/fury-core/src/main/java/io/fury/Fury.java index de98d56c6b..8269840f4f 100644 --- a/java/fury-core/src/main/java/io/fury/Fury.java +++ b/java/fury-core/src/main/java/io/fury/Fury.java @@ -32,6 +32,7 @@ import io.fury.serializer.ArraySerializers; import io.fury.serializer.BufferCallback; import io.fury.serializer.BufferObject; +import io.fury.serializer.CompatibleMode; import io.fury.serializer.JavaSerializer; import io.fury.serializer.OpaqueObjects; import io.fury.serializer.Serializer; @@ -1035,6 +1036,7 @@ public static final class FuryBuilder { ClassLoader classLoader; boolean compressNumber = false; boolean compressString = true; + CompatibleMode compatibleMode = CompatibleMode.SCHEMA_CONSISTENT; // TODO(chaokunyang) switch to object stream serializer. Class defaultJDKStreamSerializerType = JavaSerializer.class; boolean secureModeEnabled = true; @@ -1079,6 +1081,11 @@ public FuryBuilder withClassLoader(ClassLoader classLoader) { return this; } + public FuryBuilder withCompatibleMode(CompatibleMode compatibleMode) { + this.compatibleMode = compatibleMode; + return this; + } + public FuryBuilder requireClassRegistration(boolean requireClassRegistration) { this.requireClassRegistration = requireClassRegistration; return this; diff --git a/java/fury-core/src/main/java/io/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/io/fury/resolver/ClassResolver.java index f6986f62e9..bc401b91ce 100644 --- a/java/fury-core/src/main/java/io/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/io/fury/resolver/ClassResolver.java @@ -35,6 +35,7 @@ import io.fury.serializer.ArraySerializers; import io.fury.serializer.BufferSerializers; import io.fury.serializer.CollectionSerializers; +import io.fury.serializer.CompatibleSerializer; import io.fury.serializer.ExternalizableSerializer; import io.fury.serializer.JavaSerializer; import io.fury.serializer.JdkProxySerializer; @@ -636,12 +637,24 @@ public Class getSerializerClass(Class cls) { if (requireJavaSerialization(cls)) { return getJavaSerializer(cls); } - return ObjectSerializer.class; + return getObjectSerializerClass(cls); } } public Class getObjectSerializerClass(Class cls) { - return ObjectSerializer.class; + if (fury.getLanguage() != Language.JAVA) { + LOG.warn("Class {} isn't supported for cross-language serialization.", cls); + } + LOG.debug("Object of type {} can't be serialized by jit", cls); + switch (fury.getConfig().getCompatibleMode()) { + case SCHEMA_CONSISTENT: + return ObjectSerializer.class; + case COMPATIBLE: + return CompatibleSerializer.class; + default: + throw new UnsupportedOperationException( + String.format("Unsupported mode %s", fury.getConfig().getCompatibleMode())); + } } public Class getJavaSerializer(Class clz) { diff --git a/java/fury-core/src/main/java/io/fury/resolver/FieldResolver.java b/java/fury-core/src/main/java/io/fury/resolver/FieldResolver.java index 2c95b84ed5..4e1d6da499 100644 --- a/java/fury-core/src/main/java/io/fury/resolver/FieldResolver.java +++ b/java/fury-core/src/main/java/io/fury/resolver/FieldResolver.java @@ -100,6 +100,7 @@ * info is less than current field, then it will be a field not exists in current class and can be * skipped. * + * @see io.fury.serializer.CompatibleSerializerBase * @author chaokunyang */ @SuppressWarnings({"rawtypes", "UnstableApiUsage"}) diff --git a/java/fury-core/src/main/java/io/fury/serializer/CompatibleSerializer.java b/java/fury-core/src/main/java/io/fury/serializer/CompatibleSerializer.java new file mode 100644 index 0000000000..56bccc59f6 --- /dev/null +++ b/java/fury-core/src/main/java/io/fury/serializer/CompatibleSerializer.java @@ -0,0 +1,600 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.serializer; + +import com.google.common.base.Preconditions; +import io.fury.Fury; +import io.fury.memory.MemoryBuffer; +import io.fury.resolver.ClassInfo; +import io.fury.resolver.ClassResolver; +import io.fury.resolver.FieldResolver; +import io.fury.resolver.ReferenceResolver; +import io.fury.util.Platform; +import io.fury.util.UnsafeFieldAccessor; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Collection; +import java.util.Map; + +/** + * This Serializer provides both forward and backward compatibility: fields can be added or removed + * without invalidating previously serialized bytes. + * + * @see FieldResolver + * @author chaokunyang + */ +// TODO(chaokunyang) support generics optimization for {@code SomeClass} +@SuppressWarnings({"unchecked", "rawtypes"}) +public final class CompatibleSerializer extends CompatibleSerializerBase { + private static final int INDEX_FOR_SKIP_FILL_VALUES = -1; + private final ReferenceResolver referenceResolver; + private final ClassResolver classResolver; + private final FieldResolver fieldResolver; + private final Constructor constructor; + private final boolean compressNumber; + + public CompatibleSerializer(Fury fury, Class cls) { + super(fury, cls); + this.referenceResolver = fury.getReferenceResolver(); + this.classResolver = fury.getClassResolver(); + // Use `setSerializerIfAbsent` to avoid overwriting existing serializer for class when used + // as data serializer. + classResolver.setSerializerIfAbsent(cls, this); + Constructor constructor; + try { + constructor = cls.getConstructor(); + if (!constructor.isAccessible()) { + constructor.setAccessible(true); + } + } catch (Exception e) { + constructor = null; + } + this.constructor = constructor; + fieldResolver = classResolver.getFieldResolver(cls); + compressNumber = fury.compressNumber(); + } + + public CompatibleSerializer(Fury fury, Class cls, FieldResolver fieldResolver) { + super(fury, cls); + this.referenceResolver = fury.getReferenceResolver(); + this.classResolver = fury.getClassResolver(); + this.constructor = null; + this.fieldResolver = fieldResolver; + compressNumber = fury.compressNumber(); + } + + @Override + public void write(MemoryBuffer buffer, T value) { + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getEmbedTypes4Fields()) { + buffer.writeInt((int) fieldInfo.getEncodedFieldInfo()); + readAndWriteFieldValue(buffer, fieldInfo, value); + } + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getEmbedTypes9Fields()) { + buffer.writeLong(fieldInfo.getEncodedFieldInfo()); + readAndWriteFieldValue(buffer, fieldInfo, value); + } + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getEmbedTypesHashFields()) { + buffer.writeLong(fieldInfo.getEncodedFieldInfo()); + readAndWriteFieldValue(buffer, fieldInfo, value); + } + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getSeparateTypesHashFields()) { + buffer.writeLong(fieldInfo.getEncodedFieldInfo()); + readAndWriteFieldValue(buffer, fieldInfo, value); + } + buffer.writeLong(fieldResolver.getEndTag()); + } + + public void writeFieldsValues(MemoryBuffer buffer, Object[] vals) { + FieldResolver fieldResolver = this.fieldResolver; + Fury fury = this.fury; + int index = 0; + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getEmbedTypes4Fields()) { + buffer.writeInt((int) fieldInfo.getEncodedFieldInfo()); + writeFieldValue(fieldInfo, buffer, vals[index++]); + } + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getEmbedTypes9Fields()) { + buffer.writeLong(fieldInfo.getEncodedFieldInfo()); + writeFieldValue(fieldInfo, buffer, vals[index++]); + } + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getEmbedTypesHashFields()) { + buffer.writeLong(fieldInfo.getEncodedFieldInfo()); + writeFieldValue(fieldInfo, buffer, vals[index++]); + } + for (FieldResolver.FieldInfo fieldInfo : fieldResolver.getSeparateTypesHashFields()) { + buffer.writeLong(fieldInfo.getEncodedFieldInfo()); + Object value = vals[index++]; + if (!fury.getReferenceResolver().writeReferenceOrNull(buffer, value)) { + byte fieldType = fieldInfo.getFieldType(); + buffer.writeByte(fieldType); + Preconditions.checkArgument(fieldType == FieldResolver.FieldTypes.OBJECT); + ClassInfo classInfo = fieldInfo.getClassInfo(value.getClass()); + fury.writeNonReferenceToJava(buffer, value, classInfo); + } + } + buffer.writeLong(fieldResolver.getEndTag()); + } + + private void readAndWriteFieldValue( + MemoryBuffer buffer, FieldResolver.FieldInfo fieldInfo, Object targetObject) { + UnsafeFieldAccessor fieldAccessor = fieldInfo.getUnsafeFieldAccessor(); + short classId = fieldInfo.getEmbeddedClassId(); + if (ObjectSerializer.writePrimitiveFieldValueFailed( + fury, buffer, targetObject, fieldAccessor, classId)) { + Object fieldValue = fieldAccessor.getObject(targetObject); + if (ObjectSerializer.writeBasicObjectFieldValueFailed(fury, buffer, fieldValue, classId)) { + if (classId == ClassResolver.NO_CLASS_ID) { // SEPARATE_TYPES_HASH + writeSeparateFieldValue(fieldInfo, buffer, fieldValue); + } else { + ClassInfo classInfo = fieldInfo.getClassInfo(classId); + Serializer serializer = classInfo.getSerializer(); + fury.writeReferencableToJava(buffer, fieldValue, serializer); + } + } + } + } + + private void writeFieldValue( + FieldResolver.FieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + short classId = fieldInfo.getEmbeddedClassId(); + // PRIMITIVE fields, not need for null check. + switch (classId) { + case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: + buffer.writeBoolean((Boolean) fieldValue); + return; + case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: + buffer.writeByte((Byte) fieldValue); + return; + case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: + buffer.writeChar((Character) fieldValue); + return; + case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: + buffer.writeShort((Short) fieldValue); + return; + case ClassResolver.PRIMITIVE_INT_CLASS_ID: + if (compressNumber) { + buffer.writeVarInt((Integer) fieldValue); + } else { + buffer.writeInt((Integer) fieldValue); + } + return; + case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: + buffer.writeFloat((Float) fieldValue); + return; + case ClassResolver.PRIMITIVE_LONG_CLASS_ID: + if (compressNumber) { + buffer.writeVarLong((Long) fieldValue); + } else { + buffer.writeLong((Long) fieldValue); + } + return; + case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: + buffer.writeDouble((Double) fieldValue); + return; + case ClassResolver.STRING_CLASS_ID: + fury.writeJavaStringRef(buffer, (String) fieldValue); + break; + case ClassResolver.NO_CLASS_ID: // SEPARATE_TYPES_HASH + writeSeparateFieldValue(fieldInfo, buffer, fieldValue); + break; + default: + { + ClassInfo classInfo = fieldInfo.getClassInfo(classId); + Serializer serializer = classInfo.getSerializer(); + fury.writeReferencableToJava(buffer, fieldValue, serializer); + } + } + } + + private void writeSeparateFieldValue( + FieldResolver.FieldInfo fieldInfo, MemoryBuffer buffer, Object fieldValue) { + if (!referenceResolver.writeReferenceOrNull(buffer, fieldValue)) { + byte fieldType = fieldInfo.getFieldType(); + buffer.writeByte(fieldType); + if (fieldType == FieldResolver.FieldTypes.OBJECT) { + ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass()); + fury.writeNonReferenceToJava(buffer, fieldValue, classInfo); + } else { + if (fieldType == FieldResolver.FieldTypes.COLLECTION_ELEMENT_FINAL) { + writeCollectionField( + buffer, (FieldResolver.CollectionFieldInfo) fieldInfo, (Collection) fieldValue); + } else if (fieldType == FieldResolver.FieldTypes.MAP_KV_FINAL) { + writeMapKVFinal(buffer, (FieldResolver.MapFieldInfo) fieldInfo, (Map) fieldValue); + } else if (fieldType == FieldResolver.FieldTypes.MAP_KEY_FINAL) { + writeMapKeyFinal(buffer, (FieldResolver.MapFieldInfo) fieldInfo, (Map) fieldValue); + } else { + Preconditions.checkArgument(fieldType == FieldResolver.FieldTypes.MAP_VALUE_FINAL); + writeMapValueFinal(buffer, (FieldResolver.MapFieldInfo) fieldInfo, (Map) fieldValue); + } + } + } + } + + private void writeCollectionField( + MemoryBuffer buffer, FieldResolver.CollectionFieldInfo fieldInfo, Collection fieldValue) { + ClassInfo elementClassInfo = fieldInfo.getElementClassInfo(); + classResolver.writeClass(buffer, elementClassInfo); + // following write is consistent with `BaseSeqCodecBuilder.serializeForCollection` + ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass()); + classResolver.writeClass(buffer, classInfo); + CollectionSerializers.CollectionSerializer collectionSerializer = + (CollectionSerializers.CollectionSerializer) classInfo.getSerializer(); + collectionSerializer.setElementSerializer(elementClassInfo.getSerializer()); + collectionSerializer.write(buffer, fieldValue); + } + + private void writeMapKVFinal( + MemoryBuffer buffer, FieldResolver.MapFieldInfo fieldInfo, Map fieldValue) { + ClassInfo keyClassInfo = fieldInfo.getKeyClassInfo(); + ClassInfo valueClassInfo = fieldInfo.getValueClassInfo(); + classResolver.writeClass(buffer, keyClassInfo); + classResolver.writeClass(buffer, valueClassInfo); + // following write is consistent with `BaseSeqCodecBuilder.serializeForMap` + ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass()); + classResolver.writeClass(buffer, classInfo); + MapSerializers.MapSerializer mapSerializer = + (MapSerializers.MapSerializer) classInfo.getSerializer(); + mapSerializer.setKeySerializer(keyClassInfo.getSerializer()); + mapSerializer.setValueSerializer(valueClassInfo.getSerializer()); + mapSerializer.write(buffer, fieldValue); + } + + private void writeMapKeyFinal( + MemoryBuffer buffer, FieldResolver.MapFieldInfo fieldInfo, Map fieldValue) { + ClassInfo keyClassInfo = fieldInfo.getKeyClassInfo(); + classResolver.writeClass(buffer, keyClassInfo); + // following write is consistent with `BaseSeqCodecBuilder.serializeForMap` + ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass()); + classResolver.writeClass(buffer, classInfo); + MapSerializers.MapSerializer mapSerializer = + (MapSerializers.MapSerializer) classInfo.getSerializer(); + mapSerializer.setKeySerializer(keyClassInfo.getSerializer()); + mapSerializer.write(buffer, fieldValue); + } + + private void writeMapValueFinal( + MemoryBuffer buffer, FieldResolver.MapFieldInfo fieldInfo, Map fieldValue) { + ClassInfo valueClassInfo = fieldInfo.getValueClassInfo(); + classResolver.writeClass(buffer, valueClassInfo); + // following write is consistent with `BaseSeqCodecBuilder.serializeForMap` + ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass()); + classResolver.writeClass(buffer, classInfo); + MapSerializers.MapSerializer mapSerializer = + (MapSerializers.MapSerializer) classInfo.getSerializer(); + mapSerializer.setValueSerializer(valueClassInfo.getSerializer()); + mapSerializer.write(buffer, fieldValue); + } + + @SuppressWarnings("unchecked") + @Override + public T read(MemoryBuffer buffer) { + T obj = (T) newBean(); + referenceResolver.reference(obj); + return readAndSetFields(buffer, obj); + } + + @Override + public T readAndSetFields(MemoryBuffer buffer, T obj) { + long partFieldInfo = readEmbedTypes4Fields(buffer, obj, null, INDEX_FOR_SKIP_FILL_VALUES); + long endTag = fieldResolver.getEndTag(); + if (partFieldInfo == endTag) { + return obj; + } + long tmp = buffer.readInt(); + partFieldInfo = tmp << 32 | (partFieldInfo & 0x00000000ffffffffL); + partFieldInfo = + readEmbedTypes9Fields(buffer, partFieldInfo, obj, null, INDEX_FOR_SKIP_FILL_VALUES); + if (partFieldInfo == endTag) { + return obj; + } + partFieldInfo = + readEmbedTypesHashFields(buffer, partFieldInfo, obj, null, INDEX_FOR_SKIP_FILL_VALUES); + if (partFieldInfo == endTag) { + return obj; + } + readSeparateTypesHashField(buffer, partFieldInfo, obj, null, INDEX_FOR_SKIP_FILL_VALUES); + return obj; + } + + public void readFields(MemoryBuffer buffer, Object[] vals) { + int startIndex = 0; + long partFieldInfo = readEmbedTypes4Fields(buffer, null, vals, startIndex); + long endTag = fieldResolver.getEndTag(); + if (partFieldInfo == endTag) { + return; + } + startIndex += fieldResolver.getEmbedTypes4Fields().length; + long tmp = buffer.readInt(); + partFieldInfo = tmp << 32 | (partFieldInfo & 0x00000000ffffffffL); + partFieldInfo = readEmbedTypes9Fields(buffer, partFieldInfo, null, vals, startIndex); + if (partFieldInfo == endTag) { + return; + } + startIndex += fieldResolver.getEmbedTypes9Fields().length; + partFieldInfo = readEmbedTypesHashFields(buffer, partFieldInfo, null, vals, startIndex); + if (partFieldInfo == endTag) { + return; + } + startIndex += fieldResolver.getEmbedTypesHashFields().length; + readSeparateTypesHashField(buffer, partFieldInfo, null, vals, startIndex); + } + + private long readEmbedTypes4Fields( + MemoryBuffer buffer, Object obj, Object[] vals, int startIndex) { + long partFieldInfo = buffer.readInt(); + FieldResolver.FieldInfo[] embedTypes4Fields = fieldResolver.getEmbedTypes4Fields(); + if (embedTypes4Fields.length > 0) { + long minFieldInfo = embedTypes4Fields[0].getEncodedFieldInfo(); + while ((partFieldInfo & 0b11) == FieldResolver.EMBED_TYPES_4_FLAG + && partFieldInfo < minFieldInfo) { + long part = fieldResolver.skipDataBy4(buffer, (int) partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readInt(); + } + for (int i = 0; i < embedTypes4Fields.length; i++) { + FieldResolver.FieldInfo fieldInfo = embedTypes4Fields[i]; + long encodedFieldInfo = fieldInfo.getEncodedFieldInfo(); + if (encodedFieldInfo == partFieldInfo) { + if (obj != null) { + readAndSetFieldValue(fieldInfo, buffer, obj); + } else { + vals[startIndex + i] = readFieldValue(fieldInfo, buffer); + } + partFieldInfo = buffer.readInt(); + } else { + if ((partFieldInfo & 0b11) == FieldResolver.EMBED_TYPES_4_FLAG) { + if (partFieldInfo < encodedFieldInfo) { + long part = fieldResolver.skipDataBy4(buffer, (int) partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readInt(); + i--; + } + } else { + break; + } + } + } + } + while ((partFieldInfo & 0b11) == FieldResolver.EMBED_TYPES_4_FLAG) { + long part = fieldResolver.skipDataBy4(buffer, (int) partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readInt(); + } + return partFieldInfo; + } + + private long readEmbedTypes9Fields( + MemoryBuffer buffer, long partFieldInfo, Object obj, Object[] vals, int startIndex) { + FieldResolver.FieldInfo[] embedTypes9Fields = fieldResolver.getEmbedTypes9Fields(); + if (embedTypes9Fields.length > 0) { + long minFieldInfo = embedTypes9Fields[0].getEncodedFieldInfo(); + while ((partFieldInfo & 0b111) == FieldResolver.EMBED_TYPES_9_FLAG + && partFieldInfo < minFieldInfo) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readLong(); + } + + for (int i = 0; i < embedTypes9Fields.length; i++) { + FieldResolver.FieldInfo fieldInfo = embedTypes9Fields[i]; + long encodedFieldInfo = fieldInfo.getEncodedFieldInfo(); + if (encodedFieldInfo == partFieldInfo) { + if (obj != null) { + readAndSetFieldValue(fieldInfo, buffer, obj); + } else { + vals[startIndex + i] = readFieldValue(fieldInfo, buffer); + } + partFieldInfo = buffer.readLong(); + } else { + if ((partFieldInfo & 0b111) == FieldResolver.EMBED_TYPES_9_FLAG) { + if (partFieldInfo < encodedFieldInfo) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readLong(); + i--; + } + } else { + break; + } + } + } + } + while ((partFieldInfo & 0b111) == FieldResolver.EMBED_TYPES_9_FLAG) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readLong(); + } + return partFieldInfo; + } + + private long readEmbedTypesHashFields( + MemoryBuffer buffer, long partFieldInfo, Object obj, Object[] vals, int startIndex) { + FieldResolver.FieldInfo[] embedTypesHashFields = fieldResolver.getEmbedTypesHashFields(); + if (embedTypesHashFields.length > 0) { + long minFieldInfo = embedTypesHashFields[0].getEncodedFieldInfo(); + while ((partFieldInfo & 0b111) == FieldResolver.EMBED_TYPES_HASH_FLAG + && partFieldInfo < minFieldInfo) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readLong(); + } + + for (int i = 0; i < embedTypesHashFields.length; i++) { + FieldResolver.FieldInfo fieldInfo = embedTypesHashFields[i]; + long encodedFieldInfo = fieldInfo.getEncodedFieldInfo(); + if (encodedFieldInfo == partFieldInfo) { + if (obj != null) { + readAndSetFieldValue(fieldInfo, buffer, obj); + } else { + vals[startIndex + i] = readFieldValue(fieldInfo, buffer); + } + partFieldInfo = buffer.readLong(); + } else { + if ((partFieldInfo & 0b111) == FieldResolver.EMBED_TYPES_HASH_FLAG) { + if (partFieldInfo < encodedFieldInfo) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readLong(); + i--; + } + } else { + break; + } + } + } + } + while ((partFieldInfo & 0b111) == FieldResolver.EMBED_TYPES_HASH_FLAG) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return part; + } + partFieldInfo = buffer.readLong(); + } + return partFieldInfo; + } + + private void readSeparateTypesHashField( + MemoryBuffer buffer, long partFieldInfo, Object obj, Object[] vals, int startIndex) { + FieldResolver.FieldInfo[] separateTypesHashFields = fieldResolver.getSeparateTypesHashFields(); + if (separateTypesHashFields.length > 0) { + long minFieldInfo = separateTypesHashFields[0].getEncodedFieldInfo(); + while ((partFieldInfo & 0b11) == FieldResolver.SEPARATE_TYPES_HASH_FLAG + && partFieldInfo < minFieldInfo) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return; + } + partFieldInfo = buffer.readLong(); + } + for (int i = 0; i < separateTypesHashFields.length; i++) { + FieldResolver.FieldInfo fieldInfo = separateTypesHashFields[i]; + long encodedFieldInfo = fieldInfo.getEncodedFieldInfo(); + if (encodedFieldInfo == partFieldInfo) { + if (obj != null) { + readAndSetFieldValue(fieldInfo, buffer, obj); + } else { + vals[startIndex + i] = readFieldValue(fieldInfo, buffer); + } + partFieldInfo = buffer.readLong(); + } else { + if ((partFieldInfo & 0b11) == FieldResolver.SEPARATE_TYPES_HASH_FLAG) { + if (partFieldInfo < encodedFieldInfo) { + long part = fieldResolver.skipDataBy8(buffer, partFieldInfo); + if (part != partFieldInfo) { + return; + } + partFieldInfo = buffer.readLong(); + i--; + } + } else { + break; + } + } + } + } + fieldResolver.skipEndFields(buffer, partFieldInfo); + } + + private void readAndSetFieldValue( + FieldResolver.FieldInfo fieldInfo, MemoryBuffer buffer, Object targetObject) { + UnsafeFieldAccessor fieldAccessor = fieldInfo.getUnsafeFieldAccessor(); + short classId = fieldInfo.getEmbeddedClassId(); + if (ObjectSerializer.readPrimitiveFieldValueFailed( + fury, buffer, targetObject, fieldAccessor, classId) + && ObjectSerializer.readBasicObjectFieldValueFailed( + fury, buffer, targetObject, fieldAccessor, classId)) { + if (classId == ClassResolver.NO_CLASS_ID) { + // SEPARATE_TYPES_HASH + Object fieldValue = fieldResolver.readObjectField(buffer, fieldInfo); + fieldAccessor.putObject(targetObject, fieldValue); + } else { + ClassInfo classInfo = fieldInfo.getClassInfo(classId); + Serializer serializer = classInfo.getSerializer(); + fieldAccessor.putObject(targetObject, fury.readReferencableFromJava(buffer, serializer)); + } + } + } + + private Object readFieldValue(FieldResolver.FieldInfo fieldInfo, MemoryBuffer buffer) { + short classId = fieldInfo.getEmbeddedClassId(); + // PRIMITIVE fields, not need for null check. + switch (classId) { + case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: + return buffer.readBoolean(); + case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: + return buffer.readByte(); + case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: + return buffer.readChar(); + case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: + return buffer.readShort(); + case ClassResolver.PRIMITIVE_INT_CLASS_ID: + if (compressNumber) { + return buffer.readVarInt(); + } else { + return buffer.readInt(); + } + case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: + return buffer.readFloat(); + case ClassResolver.PRIMITIVE_LONG_CLASS_ID: + if (compressNumber) { + return buffer.readVarLong(); + } else { + return buffer.readLong(); + } + case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: + return buffer.readDouble(); + case ClassResolver.STRING_CLASS_ID: + return fury.readJavaStringRef(buffer); + case ClassResolver.NO_CLASS_ID: + return fieldResolver.readObjectField(buffer, fieldInfo); + default: + { + ClassInfo classInfo = fieldInfo.getClassInfo(classId); + Serializer serializer = classInfo.getSerializer(); + return fury.readReferencableFromJava(buffer, serializer); + } + } + } + + private Object newBean() { + if (constructor != null) { + try { + return constructor.newInstance(); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + Platform.throwException(e); + } + } + return Platform.newInstance(type); + } +} diff --git a/java/fury-core/src/test/java/io/fury/FuryTestBase.java b/java/fury-core/src/test/java/io/fury/FuryTestBase.java index bbf975ee5b..9bb45560ce 100644 --- a/java/fury-core/src/test/java/io/fury/FuryTestBase.java +++ b/java/fury-core/src/test/java/io/fury/FuryTestBase.java @@ -59,6 +59,16 @@ public static Object[][] endian() { return new Object[][] {{false}, {true}}; } + @DataProvider(name = "compressNumber") + public static Object[][] compressNumber() { + return new Object[][] {{false}, {true}}; + } + + @DataProvider(name = "refTrackingAndCompressNumber") + public static Object[][] refTrackingAndCompressNumber() { + return new Object[][] {{false, false}, {true, false}, {false, true}, {true, true}}; + } + @DataProvider(name = "crossLanguageReferenceTrackingConfig") public static Object[][] crossLanguageReferenceTrackingConfig() { return new Object[][] { diff --git a/java/fury-core/src/test/java/io/fury/serializer/CompatibleSerializerTest.java b/java/fury-core/src/test/java/io/fury/serializer/CompatibleSerializerTest.java new file mode 100644 index 0000000000..0ce3a09663 --- /dev/null +++ b/java/fury-core/src/test/java/io/fury/serializer/CompatibleSerializerTest.java @@ -0,0 +1,474 @@ +/* + * Copyright 2023 The Fury authors + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fury.serializer; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.fury.Fury; +import io.fury.FuryTestBase; +import io.fury.Language; +import io.fury.test.bean.BeanA; +import io.fury.test.bean.BeanB; +import io.fury.test.bean.CollectionFields; +import io.fury.test.bean.Foo; +import io.fury.test.bean.MapFields; +import io.fury.test.bean.Struct; +import io.fury.util.ClassLoaderUtils; +import io.fury.util.ReflectionUtils; +import java.io.ByteArrayOutputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import lombok.Data; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class CompatibleSerializerTest extends FuryTestBase { + @Test(dataProvider = "referenceTrackingConfig") + public void testWrite(boolean referenceTracking) { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTracking) + .withClassRegistrationRequired(false) + .build(); + fury.registerSerializer(Foo.class, new CompatibleSerializer<>(fury, Foo.class)); + fury.registerSerializer(BeanA.class, new CompatibleSerializer<>(fury, BeanA.class)); + fury.registerSerializer(BeanB.class, new CompatibleSerializer<>(fury, BeanB.class)); + serDeCheck(fury, Foo.create()); + serDeCheck(fury, BeanB.createBeanB(2)); + serDeCheck(fury, BeanA.createBeanA(2)); + } + + @Test(dataProvider = "referenceTrackingConfig") + public void testWriteCompatibleBasic(boolean referenceTrackingConfig) throws Exception { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + fury.registerSerializer(Foo.class, new CompatibleSerializer<>(fury, Foo.class)); + Object foo = Foo.create(); + for (Class fooClass : + new Class[] { + Foo.createCompatibleClass1(), Foo.createCompatibleClass2(), Foo.createCompatibleClass3(), + }) { + Object newFoo = fooClass.newInstance(); + ReflectionUtils.unsafeCopy(foo, newFoo); + Fury newFury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .withClassLoader(fooClass.getClassLoader()) + .build(); + newFury.registerSerializer(fooClass, new CompatibleSerializer<>(newFury, fooClass)); + { + byte[] foo1Bytes = newFury.serialize(newFoo); + Object deserialized = fury.deserialize(foo1Bytes); + Assert.assertEquals(deserialized.getClass(), Foo.class); + Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(deserialized, newFoo)); + byte[] fooBytes = fury.serialize(deserialized); + Assert.assertTrue( + ReflectionUtils.objectFieldsEquals(newFury.deserialize(fooBytes), newFoo)); + } + { + byte[] bytes1 = fury.serialize(foo); + Object o1 = newFury.deserialize(bytes1); + Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(o1, foo)); + Object o2 = fury.deserialize(newFury.serialize(o1)); + List fields = + Arrays.stream(fooClass.getDeclaredFields()) + .map(f -> f.getDeclaringClass().getSimpleName() + f.getType() + f.getName()) + .collect(Collectors.toList()); + Assert.assertTrue(ReflectionUtils.objectFieldsEquals(new HashSet<>(fields), o2, foo)); + } + { + Fury fury2 = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .withClassLoader(fooClass.getClassLoader()) + .build(); + fury2.registerSerializer(Foo.class, new CompatibleSerializer<>(newFury, Foo.class)); + Object o3 = fury.deserialize(newFury.serialize(foo)); + Assert.assertTrue(ReflectionUtils.objectFieldsEquals(o3, foo)); + } + } + } + + @Data + public static class CollectionOuter { + public List beanBList; + } + + @Test(dataProvider = "referenceTrackingConfig") + public void testWriteNestedCollection(boolean referenceTrackingConfig) throws Exception { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + fury.registerSerializer( + CollectionOuter.class, new CompatibleSerializer<>(fury, CollectionOuter.class)); + fury.registerSerializer(BeanB.class, new ObjectSerializer<>(fury, BeanB.class)); + CollectionOuter collectionOuter = new CollectionOuter(); + collectionOuter.beanBList = new ArrayList<>(ImmutableList.of(BeanB.createBeanB(2))); + byte[] newBeanABytes = fury.serialize(collectionOuter); + Object deserialized = fury.deserialize(newBeanABytes); + Assert.assertEquals(deserialized, collectionOuter); + } + + @Data + public static class MapOuter { + public Map stringBeanBMap; + } + + @Test(dataProvider = "referenceTrackingConfig") + public void testWriteNestedMap(boolean referenceTrackingConfig) throws Exception { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + fury.registerSerializer(MapOuter.class, new CompatibleSerializer<>(fury, MapOuter.class)); + fury.registerSerializer(BeanB.class, new ObjectSerializer<>(fury, BeanB.class)); + MapOuter outerCollection = new MapOuter(); + outerCollection.stringBeanBMap = new HashMap<>(ImmutableMap.of("k", BeanB.createBeanB(2))); + byte[] newBeanABytes = fury.serialize(outerCollection); + Object deserialized = fury.deserialize(newBeanABytes); + Assert.assertEquals(deserialized, outerCollection); + } + + @Test(dataProvider = "referenceTrackingConfig") + public void testWriteCompatibleContainer(boolean referenceTrackingConfig) throws Exception { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + fury.registerSerializer(BeanA.class, new CompatibleSerializer<>(fury, BeanA.class)); + fury.registerSerializer(BeanB.class, new ObjectSerializer<>(fury, BeanB.class)); + BeanA beanA = BeanA.createBeanA(2); + Class cls = createCompatibleClass1(); + Object newBeanA = cls.newInstance(); + ReflectionUtils.unsafeCopy(beanA, newBeanA); + Fury newFury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .withClassLoader(cls.getClassLoader()) + .build(); + newFury.registerSerializer(cls, new CompatibleSerializer<>(newFury, cls)); + newFury.registerSerializer(BeanB.class, new ObjectSerializer<>(newFury, BeanB.class)); + byte[] newBeanABytes = newFury.serialize(newBeanA); + Object deserialized = fury.deserialize(newBeanABytes); + Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(deserialized, newBeanA)); + Assert.assertEquals(deserialized.getClass(), BeanA.class); + byte[] beanABytes = fury.serialize(deserialized); + Assert.assertTrue( + ReflectionUtils.objectFieldsEquals(newFury.deserialize(beanABytes), newBeanA)); + + byte[] objBytes = fury.serialize(beanA); + Object obj2 = newFury.deserialize(objBytes); + Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(obj2, newBeanA)); + } + + public static Class createCompatibleClass1() { + String pkg = BeanA.class.getPackage().getName(); + String code = + "" + + "package " + + pkg + + ";\n" + + "import java.util.*;\n" + + "import java.math.*;\n" + + "public class BeanA {\n" + + " private Float f4;\n" + + " private double f5;\n" + + " private BeanB beanB;\n" + + " private BeanB beanB_added;\n" + + " private int[] intArray;\n" + + " private int[] intArray_added;\n" + + " private byte[] bytes;\n" + + " private transient BeanB f13;\n" + + " public BigDecimal f16;\n" + + " public String f17;\n" + + " public String longStringNameField_added;\n" + + " private List doubleList;\n" + + " private Iterable beanBIterable;\n" + + " private List beanBList;\n" + + " private List beanBList_added;\n" + + " private Map stringBeanBMap;\n" + + " private Map stringStringMap_added;\n" + + " private int[][] int2DArray;\n" + + " private int[][] int2DArray_added;\n" + + "}"; + return loadClass(BeanA.class, code); + } + + @Test(dataProvider = "referenceTrackingConfig") + public void testWriteCompatibleCollection(boolean referenceTrackingConfig) throws Exception { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + fury.registerSerializer( + CollectionFields.class, new CompatibleSerializer<>(fury, CollectionFields.class)); + CollectionFields collectionFields = UnmodifiableSerializersTest.createCollectionFields(); + { + Object o = serDe(fury, collectionFields); + Object o1 = CollectionFields.copyToCanEqual(o, o.getClass().newInstance()); + Object o2 = + CollectionFields.copyToCanEqual( + collectionFields, collectionFields.getClass().newInstance()); + Assert.assertEquals(o1, o2); + } + Class cls = createCompatibleClass2(); + Object newObj = cls.newInstance(); + ReflectionUtils.unsafeCopy(collectionFields, newObj); + Fury newFury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + newFury.registerSerializer(cls, new CompatibleSerializer<>(newFury, cls)); + byte[] bytes1 = newFury.serialize(newObj); + Object deserialized = fury.deserialize(bytes1); + Assert.assertTrue( + ReflectionUtils.objectCommonFieldsEquals( + CollectionFields.copyToCanEqual(deserialized, deserialized.getClass().newInstance()), + CollectionFields.copyToCanEqual(newObj, newObj.getClass().newInstance()))); + Assert.assertEquals(deserialized.getClass(), CollectionFields.class); + byte[] bytes2 = fury.serialize(deserialized); + Object obj2 = newFury.deserialize(bytes2); + Assert.assertTrue( + ReflectionUtils.objectFieldsEquals( + CollectionFields.copyToCanEqual(obj2, obj2.getClass().newInstance()), + CollectionFields.copyToCanEqual(newObj, newObj.getClass().newInstance()))); + + byte[] objBytes = fury.serialize(collectionFields); + Object obj3 = newFury.deserialize(objBytes); + Assert.assertTrue( + ReflectionUtils.objectCommonFieldsEquals( + CollectionFields.copyToCanEqual(obj3, obj3.getClass().newInstance()), + CollectionFields.copyToCanEqual(newObj, newObj.getClass().newInstance()))); + } + + public static Class createCompatibleClass2() { + String pkg = CollectionFields.class.getPackage().getName(); + String code = + "" + + "package " + + pkg + + ";\n" + + "import java.util.*;\n" + + "public class CollectionFields {\n" + + " public Collection collection2;\n" + + " public List collection3;\n" + + " public Collection randomAccessList2;\n" + + " public List randomAccessList3;\n" + + " public Collection list;\n" + + " public Collection list2;\n" + + " public List list3;\n" + + " public Collection set2;\n" + + " public Set set3;\n" + + " public Collection sortedSet2;\n" + + " public SortedSet sortedSet3;\n" + + " public Map map;\n" + + " public Map map2;\n" + + " public SortedMap sortedMap3;" + + "}"; + return loadClass(CollectionFields.class, code); + } + + @Test(dataProvider = "referenceTrackingConfig") + public void testWriteCompatibleMap(boolean referenceTrackingConfig) throws Exception { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + fury.registerSerializer(MapFields.class, new CompatibleSerializer<>(fury, MapFields.class)); + MapFields mapFields = UnmodifiableSerializersTest.createMapFields(); + { + Object o = serDe(fury, mapFields); + Object o1 = MapFields.copyToCanEqual(o, o.getClass().newInstance()); + Object o2 = MapFields.copyToCanEqual(mapFields, mapFields.getClass().newInstance()); + Assert.assertEquals(o1, o2); + } + Class cls = createCompatibleClass3(); + Object newObj = cls.newInstance(); + ReflectionUtils.unsafeCopy(mapFields, newObj); + Fury newFury = + Fury.builder() + .withLanguage(Language.JAVA) + .withReferenceTracking(referenceTrackingConfig) + .withClassRegistrationRequired(false) + .build(); + newFury.registerSerializer(cls, new CompatibleSerializer<>(newFury, cls)); + byte[] bytes1 = newFury.serialize(newObj); + Object deserialized = fury.deserialize(bytes1); + Assert.assertTrue( + ReflectionUtils.objectCommonFieldsEquals( + MapFields.copyToCanEqual(deserialized, deserialized.getClass().newInstance()), + MapFields.copyToCanEqual(newObj, newObj.getClass().newInstance()))); + Assert.assertEquals(deserialized.getClass(), MapFields.class); + byte[] bytes2 = fury.serialize(deserialized); + Object obj2 = newFury.deserialize(bytes2); + Assert.assertTrue( + ReflectionUtils.objectFieldsEquals( + MapFields.copyToCanEqual(obj2, obj2.getClass().newInstance()), + MapFields.copyToCanEqual(newObj, newObj.getClass().newInstance()))); + + byte[] objBytes = fury.serialize(mapFields); + Object obj3 = newFury.deserialize(objBytes); + Assert.assertTrue( + ReflectionUtils.objectCommonFieldsEquals( + MapFields.copyToCanEqual(obj3, obj3.getClass().newInstance()), + MapFields.copyToCanEqual(newObj, newObj.getClass().newInstance()))); + } + + public static Class createCompatibleClass3() { + String pkg = MapFields.class.getPackage().getName(); + String code = + "" + + "package " + + pkg + + ";\n" + + "import java.util.*;\n" + + "import java.util.concurrent.*;\n" + + "public class MapFields {\n" + + " public Map map;\n" + + " public Map map2;\n" + + " public Map linkedHashMap;\n" + + " public LinkedHashMap linkedHashMap3;\n" + + " public SortedMap sortedMap;\n" + + " public SortedMap sortedMap2;\n" + + " public Map concurrentHashMap;\n" + + " public ConcurrentHashMap concurrentHashMap2;\n" + + " public ConcurrentSkipListMap skipListMap2;\n" + + " public ConcurrentSkipListMap skipListMap3;\n" + + " public EnumMap enumMap2;\n" + + " public Map emptyMap;\n" + + " public Map singletonMap;\n" + + " public Map singletonMap2;\n" + + "}"; + return loadClass(MapFields.class, code); + } + + @Test(dataProvider = "compressNumber") + public void testCompressInt(boolean compressNumber) throws Exception { + Fury fury = + Fury.builder() + .withLanguage(Language.JAVA) + .withNumberCompressed(compressNumber) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withClassRegistrationRequired(false) + .build(); + Class structClass = Struct.createNumberStructClass("CompatibleCompressIntStruct", 50); + serDeCheck(fury, Struct.createPOJO(structClass)); + } + + static Class loadClass(Class cls, String code) { + String pkg = ReflectionUtils.getPackage(cls); + Path path = Paths.get(pkg.replace(".", "/") + "/" + cls.getSimpleName() + ".java"); + try { + Files.deleteIfExists(path); + System.out.println(path.toAbsolutePath()); + path.getParent().toFile().mkdirs(); + Files.write(path, code.getBytes()); + // Use JavaCompiler because janino doesn't support generics. + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + int result = + compiler.run( + null, + new ByteArrayOutputStream(), // ignore output + System.err, + "-classpath", + System.getProperty("java.class.path"), + path.toString()); + if (result != 0) { + throw new RuntimeException(String.format("Couldn't compile code:\n %s.", code)); + } + Class clz = + new ClassLoaderUtils.ChildFirstURLClassLoader( + new URL[] {Paths.get(".").toUri().toURL()}, Struct.class.getClassLoader()) + .loadClass(cls.getName()); + Files.deleteIfExists(path); + Files.deleteIfExists(Paths.get(pkg.replace(".", "/") + "." + cls.getSimpleName() + ".class")); + Assert.assertNotEquals(clz, cls); + return clz; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + static Class loadClass(String pkg, String className, String code) { + Path path = Paths.get(pkg.replace(".", "/") + "/" + className + ".java"); + try { + Files.deleteIfExists(path); + System.out.println(path.toAbsolutePath()); + path.getParent().toFile().mkdirs(); + Files.write(path, code.getBytes()); + // Use JavaCompiler because janino doesn't support generics. + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + int result = + compiler.run( + null, + new ByteArrayOutputStream(), // ignore output + System.err, + "-classpath", + System.getProperty("java.class.path"), + path.toString()); + if (result != 0) { + throw new RuntimeException(String.format("Couldn't compile code:\n %s.", code)); + } + Class clz = + new ClassLoaderUtils.ChildFirstURLClassLoader( + new URL[] {Paths.get(".").toUri().toURL()}, Struct.class.getClassLoader()) + .loadClass(pkg + "." + className); + Files.deleteIfExists(path); + Files.deleteIfExists(Paths.get(pkg.replace(".", "/") + "." + className + ".class")); + return clz; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java b/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java index 583a59c5ae..c1a8b3c9e4 100644 --- a/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java +++ b/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java @@ -80,6 +80,39 @@ public static Class createCompatibleClass1() { return loadFooClass(pkg, code); } + public static Class createCompatibleClass2() { + String pkg = Foo.class.getPackage().getName(); + String code = + "" + + "package " + + pkg + + ";\n" + + "public class Foo {\n" + + " long f13;\n" + + " long f14;\n" + + " long f15;\n" + + "}"; + return loadFooClass(pkg, code); + } + + public static Class createCompatibleClass3() { + String pkg = Foo.class.getPackage().getName(); + String code = + "" + + "package " + + pkg + + ";\n" + + "public class Foo {\n" + + " int f2;\n" + + " long f4;\n" + + " float f5;\n" + + " double f6;\n" + + " long f8;\n" + + " long f14;\n" + + "}"; + return loadFooClass(pkg, code); + } + private static Class loadFooClass(String pkg, String code) { SimpleCompiler compiler = new SimpleCompiler(); compiler.setParentClassLoader(Foo.class.getClassLoader().getParent()); From 90ff77c8b1031f04e7da1ac1059424fb445339d2 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Thu, 11 May 2023 10:47:43 +0800 Subject: [PATCH 2/2] add doc --- java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java b/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java index c1a8b3c9e4..6bf42fbe07 100644 --- a/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java +++ b/java/fury-test-core/src/main/java/io/fury/test/bean/Foo.java @@ -24,6 +24,11 @@ import lombok.Data; import org.codehaus.janino.SimpleCompiler; +/** + * Test struct for primitive fields. + * + * @author chaokunyang + */ @Data public class Foo implements Serializable { int f1;