diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/CompatibleExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/CompatibleExample.java index 4a7100b75f..060d82f1be 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/CompatibleExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/CompatibleExample.java @@ -30,6 +30,7 @@ public class CompatibleExample { Fury.builder() .requireClassRegistration(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(false) .build(); // register and generate serializer code. fury.register(Foo.class, true); diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/Main.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/Main.java index 5563b914f4..6b51256b38 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/Main.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/Main.java @@ -27,6 +27,7 @@ public class Main { public static void main(String[] args) throws Throwable { Example.main(args); CompatibleExample.main(args); + ScopedCompatibleExample.main(args); RecordExample.main(args); CompatibleRecordExample.main(args); RecordExample2.main(args); diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/ScopedCompatibleExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/ScopedCompatibleExample.java new file mode 100644 index 0000000000..48883fbe72 --- /dev/null +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fury/graalvm/ScopedCompatibleExample.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.graalvm; + +import org.apache.fury.Fury; +import org.apache.fury.config.CompatibleMode; + +public class ScopedCompatibleExample { + static Fury fury; + + static { + fury = + Fury.builder() + .requireClassRegistration(true) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(true) + .build(); + // register and generate serializer code. + fury.register(Foo.class, true); + } + + public static void main(String[] args) { + Example.test(fury); + System.out.println("CompatibleExample succeed"); + } +} diff --git a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java b/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java index daa133e449..8922ced02e 100644 --- a/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java +++ b/integration_tests/latest_jdk_tests/src/test/java/org/apache/fury/integration_tests/RecordSerializersTest.java @@ -111,11 +111,12 @@ public void testRecordCompatible(boolean codegen) throws Throwable { Fury.builder() .requireClassRegistration(false) .withCodegen(codegen) + .withClassLoader(cls1.getClassLoader()) .withCompatibleMode(CompatibleMode.COMPATIBLE) .build(); byte[] bytes1 = fury.serialize(record1); Object o = fury.deserialize(bytes1); - Assert.assertEquals(record1, o); + Assert.assertEquals(o, record1); String code2 = "import java.util.*;" + "public record TestRecord(int f1, String f2, char f4, Map f5) {}"; @@ -128,10 +129,11 @@ public void testRecordCompatible(boolean codegen) throws Throwable { Fury.builder() .requireClassRegistration(false) .withCodegen(codegen) + .withClassLoader(cls2.getClassLoader()) .withCompatibleMode(CompatibleMode.COMPATIBLE) .build(); Object o2 = fury2.deserialize(fury2.serialize(record2)); - Assert.assertEquals(record2, o2); + Assert.assertEquals(o2, record2); // test compatible Assert.assertEquals(fury2.deserialize(bytes1), record2); } diff --git a/java/benchmark/src/test/java/org/apache/fury/benchmark/state/JsonTest.java b/java/benchmark/src/test/java/org/apache/fury/benchmark/state/JsonTest.java index 6f369dacca..5a1736fe6e 100644 --- a/java/benchmark/src/test/java/org/apache/fury/benchmark/state/JsonTest.java +++ b/java/benchmark/src/test/java/org/apache/fury/benchmark/state/JsonTest.java @@ -73,7 +73,6 @@ public void testSerializeJson( .withScopedMetaShare(scoped) .withCodegen(codegen) .registerGuavaTypes(false) - .withCompatibleMode(CompatibleMode.COMPATIBLE) .build(); byte[] serialized = fury.serialize(resp); DemoResponse o = (DemoResponse) fury.deserialize(serialized); diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java index 4a85a7bcc1..fcfb86da90 100644 --- a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java @@ -41,6 +41,7 @@ import org.apache.fury.type.Descriptor; import org.apache.fury.type.DescriptorGrouper; import org.apache.fury.util.ExceptionUtils; +import org.apache.fury.util.GraalvmSupport; import org.apache.fury.util.Preconditions; import org.apache.fury.util.StringUtils; import org.apache.fury.util.record.RecordComponent; @@ -136,6 +137,10 @@ protected void addCommonImports() { @SuppressWarnings({"unchecked", "rawtypes"}) public static Serializer setCodegenSerializer( Fury fury, Class cls, GeneratedMetaSharedSerializer s) { + if (GraalvmSupport.isGraalRuntime()) { + return fury.getJITContext() + .asyncVisitFury(f -> f.getClassResolver().getSerializer(s.getType())); + } // This method hold jit lock, so create jit serializer async to avoid block serialization. Class serializerClass = fury.getJITContext() diff --git a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java index b7c93d8761..d11b36ba1a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java @@ -70,8 +70,8 @@ public final class FuryBuilder { boolean checkJdkClassSerializable = true; Class defaultJDKStreamSerializerType = ObjectStreamSerializer.class; boolean requireClassRegistration = true; - boolean metaShareEnabled = false; - boolean scopedMetaShareEnabled = false; + Boolean metaShareEnabled; + Boolean scopedMetaShareEnabled; boolean codeGenEnabled = true; Boolean deserializeNonexistentClass; boolean asyncCompilationEnabled = false; @@ -239,6 +239,9 @@ public FuryBuilder suppressClassRegistrationWarnings(boolean suppress) { /** Whether to enable meta share mode. */ public FuryBuilder withMetaShare(boolean shareMeta) { this.metaShareEnabled = shareMeta; + if (!shareMeta) { + scopedMetaShareEnabled = false; + } return this; } @@ -248,9 +251,6 @@ public FuryBuilder withMetaShare(boolean shareMeta) { */ public FuryBuilder withScopedMetaShare(boolean scoped) { scopedMetaShareEnabled = scoped; - if (scoped) { - metaShareEnabled = true; - } return this; } @@ -330,10 +330,29 @@ private void finish() { if (deserializeNonexistentClass == null) { deserializeNonexistentClass = true; } + if (scopedMetaShareEnabled == null) { + if (metaShareEnabled == null) { + metaShareEnabled = true; + scopedMetaShareEnabled = true; + } else { + scopedMetaShareEnabled = false; + } + } else { + if (metaShareEnabled == null) { + metaShareEnabled = scopedMetaShareEnabled; + } + } } else { if (deserializeNonexistentClass == null) { deserializeNonexistentClass = false; } + if (scopedMetaShareEnabled != null) { + LOG.warn("Scoped meta share is for CompatibleMode only, disable it for {}", compatibleMode); + } + scopedMetaShareEnabled = false; + if (metaShareEnabled == null) { + metaShareEnabled = false; + } } if (!requireClassRegistration) { LOG.warn( diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java index 154654caff..cde7a50cd3 100644 --- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java +++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassDefDecoder.java @@ -79,7 +79,9 @@ public static ClassDef decodeClassDef(ClassResolver classResolver, MemoryBuffer int numFields = currentClassHeader >>> 1; if (isRegistered) { int registeredId = classDefBuf.readVarUint32Small7(); - className = classResolver.getClassInfo((short) registeredId).getCls().getName(); + Class cls = classResolver.getClassInfo((short) registeredId).getCls(); + className = cls.getName(); + classSpec = new ClassSpec(cls); } else { String pkg = readPkgName(classDefBuf); String typeName = readTypeName(classDefBuf); diff --git a/java/fury-core/src/main/java/org/apache/fury/meta/ClassSpec.java b/java/fury-core/src/main/java/org/apache/fury/meta/ClassSpec.java index 6e09d04607..d32eca8bbf 100644 --- a/java/fury-core/src/main/java/org/apache/fury/meta/ClassSpec.java +++ b/java/fury-core/src/main/java/org/apache/fury/meta/ClassSpec.java @@ -19,6 +19,8 @@ package org.apache.fury.meta; +import org.apache.fury.type.TypeUtils; + public class ClassSpec { public final String entireClassName; @@ -28,6 +30,13 @@ public class ClassSpec { public final boolean isArray; public final int dimension; + public ClassSpec(Class cls) { + this.entireClassName = cls.getName(); + isEnum = cls.isEnum(); + isArray = cls.isArray(); + dimension = isArray ? TypeUtils.getArrayDimensions(cls) : 0; + } + public ClassSpec(String entireClassName, boolean isEnum, boolean isArray, int dimension) { this.entireClassName = entireClassName; this.isEnum = isEnum; diff --git a/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java b/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java index 18357000fe..53610ead5b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java +++ b/java/fury-core/src/main/java/org/apache/fury/reflect/ReflectionUtils.java @@ -34,6 +34,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -696,8 +697,8 @@ private static boolean objectCommonFieldsEquals( Object o2) { for (String commonField : commonFieldsInfo.f0) { - Field field1 = commonFieldsInfo.f1.get(commonField); - Field field2 = commonFieldsInfo.f2.get(commonField); + Field field1 = Objects.requireNonNull(commonFieldsInfo.f1.get(commonField)); + Field field2 = Objects.requireNonNull(commonFieldsInfo.f2.get(commonField)); FieldAccessor accessor1 = FieldAccessor.createAccessor(field1); FieldAccessor accessor2 = FieldAccessor.createAccessor(field2); Object f1 = accessor1.get(o1); @@ -769,15 +770,13 @@ public static Tuple3, Map, Map> getCom .collect( Collectors.toMap( // don't use `getGenericType` since janino doesn't support generics. - f -> f.getDeclaringClass().getSimpleName() + f.getType() + f.getName(), + f -> f.getDeclaringClass().getSimpleName() + f.getName(), f -> f)); Map fieldMap2 = fields2.stream() .collect( - Collectors.toMap( - f -> f.getDeclaringClass().getSimpleName() + f.getType() + f.getName(), - f -> f)); - Set commonFields = fieldMap1.keySet(); + Collectors.toMap(f -> f.getDeclaringClass().getSimpleName() + f.getName(), f -> f)); + Set commonFields = new HashSet<>(fieldMap1.keySet()); commonFields.retainAll(fieldMap2.keySet()); return Tuple3.of(commonFields, fieldMap1, fieldMap2); } diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java index 562343c7de..79237eebe2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java @@ -24,6 +24,7 @@ import org.apache.fury.collection.Tuple2; import org.apache.fury.config.Language; +import org.apache.fury.meta.ClassDef; import org.apache.fury.meta.Encoders; import org.apache.fury.meta.MetaString.Encoding; import org.apache.fury.reflect.ReflectionUtils; @@ -36,7 +37,6 @@ * serialization. */ public class ClassInfo { - final Class cls; final MetaStringBytes fullClassNameBytes; final MetaStringBytes packageNameBytes; @@ -47,6 +47,8 @@ public class ClassInfo { // use primitive to avoid boxing // class id must be less than Integer.MAX_VALUE/2 since we use bit 0 as class id flag. short classId; + ClassDef classDef; + public boolean needToWriteClassDef; ClassInfo( Class cls, @@ -78,6 +80,7 @@ public class ClassInfo { short classId) { this.cls = cls; this.serializer = serializer; + needToWriteClassDef = serializer != null && classResolver.needToWriteClassDef(serializer); MetaStringResolver metaStringResolver = classResolver.getMetaStringResolver(); if (cls != null && classResolver.getFury().getLanguage() != Language.JAVA) { this.fullClassNameBytes = @@ -90,9 +93,7 @@ public class ClassInfo { // means only classes are serialized, not the instance. If we // serialize such class only, we need to write classname bytes. if (cls != null - && ((classId == ClassResolver.NO_CLASS_ID - && !classResolver.getFury().getConfig().isMetaShareEnabled()) - || classId == ClassResolver.REPLACE_STUB_ID)) { + && (classId == ClassResolver.NO_CLASS_ID || classId == ClassResolver.REPLACE_STUB_ID)) { // REPLACE_STUB_ID for write replace class in `ClassSerializer`. Tuple2 tuple2 = Encoders.encodePkgAndClass(cls); this.packageNameBytes = @@ -147,6 +148,11 @@ public Serializer getSerializer() { return (Serializer) serializer; } + void setSerializer(ClassResolver resolver, Serializer serializer) { + this.serializer = serializer; + needToWriteClassDef = serializer != null && resolver.needToWriteClassDef(serializer); + } + @Override public String toString() { return "ClassInfo{" diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index e8147a498f..d7aff7a45e 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -412,7 +412,14 @@ public void register(Class... classes) { public void register(Class cls, boolean createSerializer) { register(cls); if (createSerializer) { - getSerializer(cls); + ClassInfo classInfo = getClassInfo(cls); + if (metaContextShareEnabled && needToWriteClassDef(classInfo.serializer)) { + ClassDef classDef = classInfo.classDef; + if (classDef == null) { + classDef = buildClassDef(classInfo); + } + buildMetaSharedClassInfo(Tuple2.of(classDef, null), classDef); + } } } @@ -646,7 +653,7 @@ public void setSerializer(String className, Class serializ Class cls = entry.getKey(); if (cls.getName().equals(className)) { LOG.info("Clear serializer for class {}.", className); - entry.getValue().serializer = Serializers.newSerializer(fury, cls, serializer); + entry.getValue().setSerializer(this, Serializers.newSerializer(fury, cls, serializer)); classInfoCache = NIL_CLASS_INFO; return; } @@ -663,7 +670,7 @@ public void setSerializers(String classNamePrefix, Class s } if (className.startsWith(classNamePrefix)) { LOG.info("Clear serializer for class {}.", className); - entry.getValue().serializer = Serializers.newSerializer(fury, cls, serializer); + entry.getValue().setSerializer(this, Serializers.newSerializer(fury, cls, serializer)); classInfoCache = NIL_CLASS_INFO; } } @@ -687,9 +694,9 @@ public void resetSerializer(Class cls, Serializer serializer) { /** * Set serializer to avoid circular error when there is a serializer query for fields by {@link - * #getClassInfo} and {@link #getSerializer(Class)} which access current creating serializer. This - * method is used to avoid overwriting existing serializer for class when creating a data - * serializer for serialization of parts fields of a class. + * #buildMetaSharedClassInfo} and {@link #getSerializer(Class)} which access current creating + * serializer. This method is used to avoid overwriting existing serializer for class when + * creating a data serializer for serialization of parts fields of a class. */ public void setSerializerIfAbsent(Class cls, Serializer serializer) { Serializer s = getSerializer(cls, false); @@ -702,7 +709,7 @@ public void setSerializerIfAbsent(Class cls, Serializer serializer) { public void clearSerializer(Class cls) { ClassInfo classInfo = classInfoMap.get(cls); if (classInfo != null) { - classInfo.serializer = null; + classInfo.setSerializer(this, null); } } @@ -747,7 +754,7 @@ private void addSerializer(Class type, Serializer serializer) { } // 2. Set `Serializer` for `ClassInfo`. - classInfo.serializer = serializer; + classInfo.setSerializer(this, serializer); } @SuppressWarnings("unchecked") @@ -1253,83 +1260,75 @@ public void writeClassAndUpdateCache(MemoryBuffer buffer, Class cls) { /** Write classname for java serialization. */ public void writeClass(MemoryBuffer buffer, ClassInfo classInfo) { - if (classInfo.classId == NO_CLASS_ID) { // no class id provided. - // use classname - if (metaContextShareEnabled) { - buffer.writeByte(USE_CLASS_VALUE_FLAG); - // FIXME(chaokunyang) Register class but not register serializer can't be used with - // meta share mode, because no class def are sent to peer. - writeClassWithMetaShare(buffer, classInfo); - } else { + if (metaContextShareEnabled) { + // FIXME(chaokunyang) Register class but not register serializer can't be used with + // meta share mode, because no class def are sent to peer. + writeClassWithMetaShare(buffer, classInfo); + } else { + if (classInfo.classId == NO_CLASS_ID) { // no class id provided. + // use classname // if it's null, it's a bug. assert classInfo.packageNameBytes != null; metaStringResolver.writeMetaStringBytesWithFlag(buffer, classInfo.packageNameBytes); assert classInfo.classNameBytes != null; metaStringResolver.writeMetaStringBytes(buffer, classInfo.classNameBytes); + } else { + // use classId + buffer.writeVarUint32(classInfo.classId << 1); } - } else { - // use classId - buffer.writeVarUint32(classInfo.classId << 1); } } public void writeClassWithMetaShare(MemoryBuffer buffer, ClassInfo classInfo) { + if (classInfo.classId != NO_CLASS_ID && !classInfo.needToWriteClassDef) { + buffer.writeVarUint32(classInfo.classId << 1); + return; + } MetaContext metaContext = fury.getSerializationContext().getMetaContext(); Preconditions.checkNotNull( metaContext, - "Meta context must be set before serialization," - + " please set meta context by SerializationContext.setMetaContext"); + "Meta context must be set before serialization, " + + "please set meta context by SerializationContext.setMetaContext"); IdentityObjectIntMap> classMap = metaContext.classMap; int newId = classMap.size; int id = classMap.putOrGet(classInfo.cls, newId); if (id >= 0) { - buffer.writeVarUint32(id); + buffer.writeVarUint32(id << 1 | 0b1); } else { - buffer.writeVarUint32(newId); - ClassDef classDef; - Serializer serializer = classInfo.serializer; - Preconditions.checkArgument(serializer.getClass() != NonexistentClassSerializer.class); - if (fury.getConfig().getCompatibleMode() == CompatibleMode.COMPATIBLE - && (serializer instanceof Generated.GeneratedObjectSerializer - // May already switched to MetaSharedSerializer when update class info cache. - || serializer instanceof Generated.GeneratedMetaSharedSerializer - || serializer instanceof LazyInitBeanSerializer - || serializer instanceof ObjectSerializer - || serializer instanceof MetaSharedSerializer)) { - classDef = - classDefMap.computeIfAbsent(classInfo.cls, cls -> ClassDef.buildClassDef(fury, cls)); - } else { - // Some type will use other serializers such MapSerializer and so on. - classDef = - classDefMap.computeIfAbsent( - classInfo.cls, cls -> ClassDef.buildClassDef(this, cls, new ArrayList<>(), false)); + buffer.writeVarUint32(newId << 1 | 0b1); + ClassDef classDef = classInfo.classDef; + if (classDef == null) { + classDef = buildClassDef(classInfo); } metaContext.writingClassDefs.add(classDef); } } - private Class readClassWithMetaShare(MemoryBuffer buffer) { - MetaContext metaContext = fury.getSerializationContext().getMetaContext(); - Preconditions.checkNotNull( - metaContext, - "Meta context must be set before serialization," - + " please set meta context by SerializationContext.setMetaContext"); - int id = buffer.readVarUint32Small14(); - List readClassInfos = metaContext.readClassInfos; - ClassInfo classInfo = readClassInfos.get(id); - if (classInfo == null) { - List readClassDefs = metaContext.readClassDefs; - ClassDef classDef = readClassDefs.get(id); - Class cls = loadClass(classDef.getClassSpec()); - classInfo = getClassInfo(cls, false); - if (classInfo == null) { - Short classId = extRegistry.registeredClassIdMap.get(cls); - classInfo = new ClassInfo(this, cls, null, null, classId == null ? NO_CLASS_ID : classId); - classInfoMap.put(cls, classInfo); - } - readClassInfos.set(id, classInfo); + private ClassDef buildClassDef(ClassInfo classInfo) { + ClassDef classDef; + Serializer serializer = classInfo.serializer; + Preconditions.checkArgument(serializer.getClass() != NonexistentClassSerializer.class); + if (needToWriteClassDef(serializer)) { + classDef = + classDefMap.computeIfAbsent(classInfo.cls, cls -> ClassDef.buildClassDef(fury, cls)); + } else { + // Some type will use other serializers such MapSerializer and so on. + classDef = + classDefMap.computeIfAbsent( + classInfo.cls, cls -> ClassDef.buildClassDef(this, cls, new ArrayList<>(), false)); } - return classInfo.cls; + classInfo.classDef = classDef; + return classDef; + } + + boolean needToWriteClassDef(Serializer serializer) { + return fury.getConfig().getCompatibleMode() == CompatibleMode.COMPATIBLE + && (serializer instanceof Generated.GeneratedObjectSerializer + // May already switched to MetaSharedSerializer when update class info cache. + || serializer instanceof Generated.GeneratedMetaSharedSerializer + || serializer instanceof LazyInitBeanSerializer + || serializer instanceof ObjectSerializer + || serializer instanceof MetaSharedSerializer); } private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, MetaContext metaContext) { @@ -1337,7 +1336,11 @@ private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, MetaContext me metaContext, "Meta context must be set before serialization," + " please set meta context by SerializationContext.setMetaContext"); - int id = buffer.readVarUint32Small14(); + int header = buffer.readVarUint32Small14(); + int id = header >>> 1; + if ((header & 0b1) == 0) { + return getOrUpdateClassInfo((short) id); + } List readClassInfos = metaContext.readClassInfos; ClassInfo classInfo = readClassInfos.get(id); if (classInfo == null) { @@ -1345,18 +1348,7 @@ private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, MetaContext me ClassDef classDef = readClassDefs.get(id); Tuple2 classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); if (classDefTuple == null || classDefTuple.f1 == null) { - if (classDefTuple != null) { - classDef = classDefTuple.f0; - } - Class cls = loadClass(classDef.getClassSpec()); - if (!classDef.isObjectType()) { - classInfo = getClassInfo(cls); - } else { - classInfo = getMetaSharedClassInfo(classDef, cls); - } - // Share serializer for same version class def to avoid too much different meta - // context take up too much memory. - putClassDef(classDef, classInfo); + classInfo = buildMetaSharedClassInfo(classDefTuple, classDef); } else { classInfo = classDefTuple.f1; } @@ -1365,6 +1357,24 @@ private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, MetaContext me return classInfo; } + private ClassInfo buildMetaSharedClassInfo( + Tuple2 classDefTuple, ClassDef classDef) { + ClassInfo classInfo; + if (classDefTuple != null) { + classDef = classDefTuple.f0; + } + Class cls = loadClass(classDef.getClassSpec()); + if (!classDef.isObjectType()) { + classInfo = getClassInfo(cls); + } else { + classInfo = getMetaSharedClassInfo(classDef, cls); + } + // Share serializer for same version class def to avoid too much different meta + // context take up too much memory. + putClassDef(classDef, classInfo); + return classInfo; + } + // TODO(chaokunyang) if ClassDef is consistent with class in this process, // use existing serializer instead. private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { @@ -1377,7 +1387,7 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { new ClassInfo(this, cls, null, null, classId == null ? NO_CLASS_ID : classId); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { if (cls == NonexistentMetaShared.class) { - classInfo.serializer = new NonexistentClassSerializer(fury, classDef); + classInfo.setSerializer(this, new NonexistentClassSerializer(fury, classDef)); // ensure `NonexistentMetaSharedClass` registered to write fixed-length class def, // so we can rewrite it in `NonexistentClassSerializer`. Preconditions.checkNotNull(classId); @@ -1391,15 +1401,27 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { return getClassInfo(cls); } Class sc = - fury.getJITContext() - .registerSerializerJITCallback( - () -> MetaSharedSerializer.class, - () -> CodecUtils.loadOrGenMetaSharedCodecClass(fury, cls, classDef), - c -> classInfo.serializer = Serializers.newSerializer(fury, cls, c)); + getMetaSharedDeserializerClassFromGraalvmRegistry(cls, classDef); + if (sc == null) { + if (GraalvmSupport.isGraalRuntime()) { + sc = MetaSharedSerializer.class; + LOG.warn( + "Can't generate class at runtime in graalvm for class def {}, use {} instead", + classDef, + sc); + } else { + sc = + fury.getJITContext() + .registerSerializerJITCallback( + () -> MetaSharedSerializer.class, + () -> CodecUtils.loadOrGenMetaSharedCodecClass(fury, cls, classDef), + c -> classInfo.setSerializer(this, Serializers.newSerializer(fury, cls, c))); + } + } if (sc == MetaSharedSerializer.class) { - classInfo.serializer = new MetaSharedSerializer(fury, cls, classDef); + classInfo.setSerializer(this, new MetaSharedSerializer(fury, cls, classDef)); } else { - classInfo.serializer = Serializers.newSerializer(fury, cls, sc); + classInfo.setSerializer(this, Serializers.newSerializer(fury, cls, sc)); } return classInfo; } @@ -1524,7 +1546,12 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { // ReplaceResolveSerializer.ReplaceStub classInfo.classId = NO_CLASS_ID; } - writeClass(buffer, classInfo); + if (classInfo.classId != NO_CLASS_ID) { + buffer.writeVarUint32(classInfo.classId << 1); + } else { + metaStringResolver.writeMetaStringBytesWithFlag(buffer, classInfo.packageNameBytes); + metaStringResolver.writeMetaStringBytes(buffer, classInfo.classNameBytes); + } classInfo.classId = classId; } @@ -1537,9 +1564,6 @@ public Class readClassInternal(MemoryBuffer buffer) { int header = buffer.readVarUint32Small14(); final ClassInfo classInfo; if ((header & 0b1) != 0) { - if (metaContextShareEnabled) { - return readClassWithMetaShare(buffer); - } MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytesWithFlag(buffer, header); MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); classInfo = loadBytesToClassInfo(packageBytes, simpleClassNameBytes); @@ -1556,23 +1580,19 @@ public Class readClassInternal(MemoryBuffer buffer) { * ClassInfo)} is faster since it use a non-global class info cache. */ public ClassInfo readClassInfo(MemoryBuffer buffer) { + if (metaContextShareEnabled) { + return readClassInfoWithMetaShare(buffer, fury.getSerializationContext().getMetaContext()); + } int header = buffer.readVarUint32Small14(); + ClassInfo classInfo; if ((header & 0b1) != 0) { - ClassInfo classInfo; - if (metaContextShareEnabled) { - classInfo = - readClassInfoWithMetaShare(buffer, fury.getSerializationContext().getMetaContext()); - } else { - classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); - } + classInfo = readClassInfoFromBytes(buffer, classInfoCache, header); classInfoCache = classInfo; - currentReadClass = classInfo.cls; - return classInfo; } else { - ClassInfo classInfo = getOrUpdateClassInfo((short) (header >> 1)); - currentReadClass = classInfo.cls; - return classInfo; + classInfo = getOrUpdateClassInfo((short) (header >> 1)); } + currentReadClass = classInfo.cls; + return classInfo; } /** @@ -1581,6 +1601,9 @@ public ClassInfo readClassInfo(MemoryBuffer buffer) { */ @CodegenInvoke public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { + if (metaContextShareEnabled) { + return readClassInfoWithMetaShare(buffer, fury.getSerializationContext().getMetaContext()); + } int header = buffer.readVarUint32Small14(); if ((header & 0b1) != 0) { return readClassInfoByCache(buffer, classInfoCache, header); @@ -1592,6 +1615,9 @@ public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfo classInfoCache) { /** Read class info, update classInfoHolder if cache not hit. */ @CodegenInvoke public ClassInfo readClassInfo(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + if (metaContextShareEnabled) { + return readClassInfoWithMetaShare(buffer, fury.getSerializationContext().getMetaContext()); + } int header = buffer.readVarUint32Small14(); if ((header & 0b1) != 0) { return readClassInfoFromBytes(buffer, classInfoHolder, header); @@ -1894,4 +1920,28 @@ private Class getSerializerClassFromGraalvmRegistry(Class< } return null; } + + private Class getMetaSharedDeserializerClassFromGraalvmRegistry( + Class cls, ClassDef classDef) { + List classResolvers = GRAALVM_REGISTRY.get(fury.getConfig().getConfigHash()); + if (classResolvers == null || classResolvers.isEmpty()) { + return null; + } + for (ClassResolver classResolver : classResolvers) { + if (classResolver != this) { + Tuple2 tuple2 = + classResolver.extRegistry.classIdToDef.get(classDef.getId()); + if (tuple2 != null && tuple2.f1 != null) { + return tuple2.f1.serializer.getClass(); + } + } + } + if (GraalvmSupport.isGraalRuntime()) { + if (Functions.isLambda(cls) || ReflectionUtils.isJdkProxy(cls)) { + return null; + } + throw new RuntimeException(String.format("Class %s is not registered", cls)); + } + return null; + } } diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java b/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java index 56d65bdc0e..ea4f8fcf66 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/SerializationContext.java @@ -64,6 +64,7 @@ public MetaContext getMetaContext() { * FuryBuilder#withMetaShare(boolean)} */ public void setMetaContext(MetaContext metaContext) { + assert !scopedMetaShareEnabled; this.metaContext = metaContext; } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java index dd92d55d25..974bef4656 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/NonexistentClassSerializers.java @@ -88,16 +88,15 @@ private void writeClassDef(MemoryBuffer buffer, NonexistentClass.NonexistentMeta // then revert written class id to write class info here, // since it's the only place to hold class def for not found class. buffer.increaseWriterIndex(-2); - buffer.writeByte(ClassResolver.USE_CLASS_VALUE_FLAG); MetaContext metaContext = fury.getSerializationContext().getMetaContext(); IdentityObjectIntMap classMap = metaContext.classMap; int newId = classMap.size; // class not exist, use class def id for identity. int id = classMap.putOrGet(value.classDef.getId(), newId); if (id >= 0) { - buffer.writeVarUint32(id); + buffer.writeVarUint32(id << 1 | 0b1); } else { - buffer.writeVarUint32(newId); + buffer.writeVarUint32(newId << 1 | 0b1); metaContext.writingClassDefs.add(value.classDef); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java index 137d3c94ca..8ec1b59f73 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ObjectStreamSerializer.java @@ -19,6 +19,8 @@ package org.apache.fury.serializer; +import static org.apache.fury.resolver.ClassResolver.NO_CLASS_ID; + import java.io.Externalizable; import java.io.IOException; import java.io.InvalidObjectException; @@ -130,7 +132,7 @@ public void write(MemoryBuffer buffer, Object value) { for (SlotsInfo slotsInfo : slotsInfos) { // create a classinfo to avoid null class bytes when class id is a // replacement id. - classResolver.writeClass(buffer, slotsInfo.classInfo); + classResolver.writeClassInternal(buffer, slotsInfo.classInfo.getCls()); StreamClassInfo streamClassInfo = slotsInfo.streamClassInfo; Method writeObjectMethod = streamClassInfo.writeObjectMethod; if (writeObjectMethod == null) { @@ -327,7 +329,7 @@ private static class SlotsInfo { public SlotsInfo(Fury fury, Class type) { this.cls = type; - classInfo = fury.getClassResolver().newClassInfo(type, null, ClassResolver.NO_CLASS_ID); + classInfo = fury.getClassResolver().newClassInfo(type, null, NO_CLASS_ID); ObjectStreamClass objectStreamClass = ObjectStreamClass.lookup(type); streamClassInfo = STREAM_CLASS_INFO_CACHE.get(type); // `putFields/writeFields` will convert to fields value to be written by diff --git a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties index 2af741d83d..e9d4a67b24 100644 --- a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties +++ b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties @@ -76,6 +76,8 @@ Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\ org.apache.fury.shaded.org.codehaus.janino.IClass,\ org.apache.fury.shaded.org.codehaus.janino.Java$ClassLiteral,\ org.apache.fury.resolver.ClassResolver$2,\ + org.apache.fury.meta.ClassDef,\ + org.apache.fury.meta.Encoders,\ org.apache.fury.shaded.org.codehaus.janino.Parser$1,\ org.apache.fury.shaded.org.codehaus.janino.Java$BinaryOperation,\ org.apache.fury.shaded.org.codehaus.janino.Java$Crement,\ diff --git a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java index 00f226b58e..4f6756fe51 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CyclicTest.java @@ -50,6 +50,7 @@ public static Object[][] fury() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // enableCodegen ImmutableSet.of(true, false), // async compilation + ImmutableSet.of(true, false), // scoped meta share ImmutableSet.of( CompatibleMode.SCHEMA_CONSISTENT, CompatibleMode.COMPATIBLE) // structFieldsRepeat ) @@ -62,7 +63,8 @@ public static Object[][] fury() { .withLanguage(Language.JAVA) .withCodegen((Boolean) c[0]) .withAsyncCompilation((Boolean) c[1]) - .withCompatibleMode((CompatibleMode) c[2]) + .withScopedMetaShare((Boolean) c[2]) + .withCompatibleMode((CompatibleMode) c[3]) .requireClassRegistration(false) }) .toArray(Object[][]::new); diff --git a/java/fury-core/src/test/java/org/apache/fury/FuryTestBase.java b/java/fury-core/src/test/java/org/apache/fury/FuryTestBase.java index 9ab77e424e..3f4cb6c17e 100644 --- a/java/fury-core/src/test/java/org/apache/fury/FuryTestBase.java +++ b/java/fury-core/src/test/java/org/apache/fury/FuryTestBase.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -100,11 +101,21 @@ public static Object[][] compressNumber() { return new Object[][] {{false}, {true}}; } + @DataProvider + public static Object[][] scopedMetaShare() { + return new Object[][] {{false}, {true}}; + } + @DataProvider public static Object[][] compressNumberAndCodeGen() { return new Object[][] {{false, false}, {true, false}, {false, true}, {true, true}}; } + @DataProvider + public static Object[][] compressNumberScopedMetaShare() { + return new Object[][] {{false, false}, {true, false}, {false, true}, {true, true}}; + } + @DataProvider public static Object[][] refTrackingAndCompressNumber() { return new Object[][] {{false, false}, {true, false}, {false, true}, {true, true}}; @@ -160,41 +171,43 @@ public static Object[][] javaFuryConfig() { @DataProvider public static Object[][] javaFuryKVCompatible() { + Supplier builder = + () -> + Fury.builder() + .withLanguage(Language.JAVA) + .requireClassRegistration(false) + .withScopedMetaShare(false); return new Object[][] { { - Fury.builder() - .withLanguage(Language.JAVA) + builder + .get() .withRefTracking(true) .withCodegen(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build() }, { - Fury.builder() - .withLanguage(Language.JAVA) + builder + .get() .withRefTracking(false) .withCodegen(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build() }, { - Fury.builder() - .withLanguage(Language.JAVA) + builder + .get() .withRefTracking(true) .withCodegen(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build() }, { - Fury.builder() - .withLanguage(Language.JAVA) + builder + .get() .withRefTracking(false) .withCodegen(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build() }, }; diff --git a/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java b/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java index 5898ecff02..33a23ce2f9 100644 --- a/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/builder/JITContextTest.java @@ -45,6 +45,7 @@ public class JITContextTest extends FuryTestBase { public static Object[][] config1() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // referenceTracking + ImmutableSet.of(true, false), // scoped meta share ImmutableSet.of(CompatibleMode.COMPATIBLE, CompatibleMode.SCHEMA_CONSISTENT)) .stream() .map(List::toArray) @@ -52,13 +53,15 @@ public static Object[][] config1() { } @Test(dataProvider = "config1", timeOut = 60_000) - public void testAsyncCompilation(boolean referenceTracking, CompatibleMode compatibleMode) + public void testAsyncCompilation( + boolean referenceTracking, boolean scopedMetaShare, CompatibleMode compatibleMode) throws InterruptedException { Fury fury = Fury.builder() .withLanguage(Language.JAVA) .withRefTracking(referenceTracking) .withCompatibleMode(compatibleMode) + .withScopedMetaShare(scopedMetaShare) .requireClassRegistration(false) .withAsyncCompilation(true) .build(); @@ -93,21 +96,25 @@ private Serializer getSerializer(Fury fury, Class cls) { @Test(dataProvider = "config1", timeOut = 60_000) public void testAsyncCompilationMetaShared( - boolean referenceTracking, CompatibleMode compatibleMode) throws InterruptedException { + boolean referenceTracking, boolean scopedMetaShare, CompatibleMode compatibleMode) + throws InterruptedException { Fury fury = Fury.builder() .withLanguage(Language.JAVA) .withRefTracking(referenceTracking) .withCompatibleMode(compatibleMode) + .withScopedMetaShare(scopedMetaShare) .requireClassRegistration(false) .withAsyncCompilation(true) .build(); BeanB beanB = BeanB.createBeanB(2); BeanA beanA = BeanA.createBeanA(2); MetaContext context = new MetaContext(); - fury.getSerializationContext().setMetaContext(context); + if (!scopedMetaShare) { + fury.getSerializationContext().setMetaContext(context); + } byte[] bytes1 = fury.serialize(beanB); - fury.getSerializationContext().setMetaContext(context); + if (!scopedMetaShare) fury.getSerializationContext().setMetaContext(context); byte[] bytes2 = fury.serialize(beanA); while (!(getSerializer(fury, BeanB.class) instanceof Generated)) { LOG.info("Waiting {} serializer to be jit.", BeanB.class); @@ -119,9 +126,9 @@ public void testAsyncCompilationMetaShared( } Assert.assertTrue(getSerializer(fury, BeanB.class) instanceof Generated); Assert.assertTrue(getSerializer(fury, BeanA.class) instanceof Generated); - fury.getSerializationContext().setMetaContext(context); + if (!scopedMetaShare) fury.getSerializationContext().setMetaContext(context); assertEquals(fury.deserialize(bytes1), beanB); - fury.getSerializationContext().setMetaContext(context); + if (!scopedMetaShare) fury.getSerializationContext().setMetaContext(context); assertEquals(fury.deserialize(bytes2), beanA); } } diff --git a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java index c1a50ea872..89435b2159 100644 --- a/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/meta/ClassDefTest.java @@ -20,6 +20,7 @@ package org.apache.fury.meta; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import com.google.common.collect.ImmutableList; import java.lang.reflect.Field; @@ -167,4 +168,12 @@ public void testContainerClass() { assertEquals(classDef1.getClassName(), classDef.getClassName()); assertEquals(classDef1, classDef); } + + @Test + public void testInterface() { + Fury fury = Fury.builder().withMetaShare(true).build(); + ClassDef classDef = ClassDef.buildClassDef(fury, Map.class); + assertTrue(classDef.getFieldsInfo().isEmpty()); + assertTrue(classDef.isObjectType()); + } } diff --git a/java/fury-core/src/test/java/org/apache/fury/resolver/ClassInfoTest.java b/java/fury-core/src/test/java/org/apache/fury/resolver/ClassInfoTest.java index 0d60c9bc79..ad189935a7 100644 --- a/java/fury-core/src/test/java/org/apache/fury/resolver/ClassInfoTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/resolver/ClassInfoTest.java @@ -20,7 +20,6 @@ package org.apache.fury.resolver; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import org.apache.fury.Fury; import org.apache.fury.config.Language; @@ -29,15 +28,6 @@ public class ClassInfoTest { @Test public void testEncodePackageNameAndTypeName() { - Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) - .requireClassRegistration(false) - .withMetaShare(true) - .build(); - ClassInfo info = fury.getClassResolver().getClassInfo(org.apache.fury.test.bean.Foo.class); - assertNull(info.packageNameBytes); - Fury fury1 = Fury.builder().withLanguage(Language.JAVA).requireClassRegistration(false).build(); ClassInfo info1 = fury1.getClassResolver().getClassInfo(org.apache.fury.test.bean.Foo.class); assertNotNull(info1.packageNameBytes); diff --git a/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java b/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java index 645c4a5af1..e75a0f7542 100644 --- a/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/resolver/MetaContextTest.java @@ -56,6 +56,7 @@ public void testShareClassDefCompatible(boolean enableCodegen) { .withLanguage(Language.JAVA) .withRefTracking(true) .withMetaShare(true) + .withScopedMetaShare(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withCodegen(enableCodegen) .requireClassRegistration(false) @@ -101,6 +102,7 @@ public void testFinalTypeWriteMeta(boolean enableCodegen) { .withLanguage(Language.JAVA) .withRefTracking(true) .withMetaShare(true) + .withScopedMetaShare(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withCodegen(enableCodegen) .requireClassRegistration(false) diff --git a/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java b/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java index 2f0f42fccf..12399057dd 100644 --- a/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/resolver/SerializationContextTest.java @@ -29,9 +29,9 @@ public class SerializationContextTest { @Test public void testSerializationContext() { - SerializationContext context = - new SerializationContext( - new Config(new FuryBuilder().withDeserializeNonexistentClass(false))); + FuryBuilder builder = new FuryBuilder().withDeserializeNonexistentClass(false); + builder.build(); // trigger finish + SerializationContext context = new SerializationContext(new Config(builder)); assertFalse(context.containsKey("A")); context.add("A", 1); assertTrue(context.containsKey("A")); diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java index 2f485f38fe..918259c74f 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/ArraySerializersTest.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import lombok.Data; import lombok.EqualsAndHashCode; import org.apache.fury.Fury; @@ -95,13 +96,14 @@ public void testMultiArraySerialization(boolean referenceTracking, Language lang @Test(dataProvider = "crossLanguageReferenceTrackingConfig") public void testPrimitiveArray(boolean referenceTracking, Language language) { - FuryBuilder builder = - Fury.builder() - .withLanguage(language) - .withRefTracking(referenceTracking) - .requireClassRegistration(false); - Fury fury1 = builder.build(); - Fury fury2 = builder.build(); + Supplier builder = + () -> + Fury.builder() + .withLanguage(language) + .withRefTracking(referenceTracking) + .requireClassRegistration(false); + Fury fury1 = builder.get().build(); + Fury fury2 = builder.get().build(); testPrimitiveArray(fury1, fury2); } diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/CodegenCompatibleSerializerTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/CodegenCompatibleSerializerTest.java index 7187236654..f29c211a51 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/CodegenCompatibleSerializerTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/CodegenCompatibleSerializerTest.java @@ -26,10 +26,12 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.fury.Fury; import org.apache.fury.FuryTestBase; import org.apache.fury.config.CompatibleMode; +import org.apache.fury.config.FuryBuilder; import org.apache.fury.config.Language; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.serializer.collection.UnmodifiableSerializersTest; @@ -48,21 +50,25 @@ public class CodegenCompatibleSerializerTest extends FuryTestBase { public static Object[][] config() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // referenceTracking + ImmutableSet.of(true, false), // scopedMetaShare ImmutableSet.of(true, false)) // enable codegen .stream() .map(List::toArray) .toArray(Object[][]::new); } + private FuryBuilder furyBuilder() { + return Fury.builder().withLanguage(Language.JAVA).requireClassRegistration(false); + } + @Test(dataProvider = "config") - public void testWrite(boolean referenceTracking, boolean enableCodegen) { + public void testWrite(boolean referenceTracking, boolean scopedMetaShare, boolean enableCodegen) { Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withCodegen(enableCodegen) + .withScopedMetaShare(scopedMetaShare) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); serDeCheck(fury, Foo.create()); serDeCheck(fury, BeanB.createBeanB(2)); @@ -70,16 +76,18 @@ public void testWrite(boolean referenceTracking, boolean enableCodegen) { } @Test(dataProvider = "config") - public void testWriteCompatibleBasic(boolean referenceTracking, boolean enableCodegen) - throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); + public void testWriteCompatibleBasic( + boolean referenceTracking, boolean scopedMetaShare, boolean enableCodegen) throws Exception { + Supplier builder = + () -> + Fury.builder() + .withLanguage(Language.JAVA) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(scopedMetaShare) + .requireClassRegistration(false); + Fury fury = builder.get().build(); Object foo = Foo.create(); for (Class fooClass : new Class[] { @@ -87,15 +95,7 @@ public void testWriteCompatibleBasic(boolean referenceTracking, boolean enableCo }) { Object newFoo = fooClass.newInstance(); ReflectionUtils.unsafeCopy(foo, newFoo); - Fury newFury = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .withClassLoader(fooClass.getClassLoader()) - .build(); + Fury newFury = builder.get().withClassLoader(fooClass.getClassLoader()).build(); { byte[] foo1Bytes = newFury.serialize(newFoo); @@ -113,7 +113,7 @@ public void testWriteCompatibleBasic(boolean referenceTracking, boolean enableCo Object o2 = fury.deserialize(newFury.serialize(o1)); List fields = Arrays.stream(fooClass.getDeclaredFields()) - .map(f -> f.getDeclaringClass().getSimpleName() + f.getType() + f.getName()) + .map(f -> f.getDeclaringClass().getSimpleName() + f.getName()) .collect(Collectors.toList()); Assert.assertTrue(ReflectionUtils.objectFieldsEquals(new HashSet<>(fields), o2, foo)); } @@ -125,17 +125,19 @@ public void testWriteCompatibleBasic(boolean referenceTracking, boolean enableCo } @Test(dataProvider = "config") - public void testWriteCompatibleCollectionBasic(boolean referenceTracking, boolean enableCodegen) - throws Exception { + public void testWriteCompatibleCollectionBasic( + boolean referenceTracking, boolean scopedMetaShare, boolean enableCodegen) throws Exception { BeanA beanA = BeanA.createBeanA(2); - Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); + Supplier builder = + () -> + Fury.builder() + .withLanguage(Language.JAVA) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(scopedMetaShare) + .requireClassRegistration(false); + Fury fury = builder.get().build(); String pkg = BeanA.class.getPackage().getName(); String code = "" @@ -154,15 +156,7 @@ public void testWriteCompatibleCollectionBasic(boolean referenceTracking, boolea BeanA.class, code, CodegenCompatibleSerializerTest.class + "testWriteCompatibleCollectionBasic_1"); - Fury fury1 = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .withClassLoader(cls1.getClassLoader()) - .build(); + Fury fury1 = builder.get().withClassLoader(cls1.getClassLoader()).build(); code = "" + "package " @@ -181,15 +175,7 @@ public void testWriteCompatibleCollectionBasic(boolean referenceTracking, boolea CodegenCompatibleSerializerTest.class + "testWriteCompatibleCollectionBasic_2"); Object newBeanA = cls2.newInstance(); ReflectionUtils.unsafeCopy(beanA, newBeanA); - Fury fury2 = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .withClassLoader(cls2.getClassLoader()) - .build(); + Fury fury2 = builder.get().withClassLoader(cls2.getClassLoader()).build(); byte[] newBeanABytes = fury2.serialize(newBeanA); Object deserialized = fury1.deserialize(newBeanABytes); Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(deserialized, newBeanA)); @@ -205,30 +191,24 @@ public void testWriteCompatibleCollectionBasic(boolean referenceTracking, boolea } @Test(dataProvider = "config") - public void testWriteCompatibleContainer(boolean referenceTracking, boolean enableCodegen) - throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); + public void testWriteCompatibleContainer( + boolean referenceTracking, boolean scopedMetaShare, boolean enableCodegen) throws Exception { + Supplier builder = + () -> + Fury.builder() + .withLanguage(Language.JAVA) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(scopedMetaShare) + .requireClassRegistration(false); + Fury fury = builder.get().build(); BeanA beanA = BeanA.createBeanA(2); serDe(fury, beanA); Class cls = ClassUtils.createCompatibleClass1(); Object newBeanA = cls.newInstance(); ReflectionUtils.unsafeCopy(beanA, newBeanA); - Fury newFury = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .withClassLoader(cls.getClassLoader()) - .build(); + Fury newFury = builder.get().withClassLoader(cls.getClassLoader()).build(); byte[] newBeanABytes = newFury.serialize(newBeanA); BeanA deserialized = (BeanA) fury.deserialize(newBeanABytes); Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(deserialized, newBeanA)); @@ -240,21 +220,22 @@ public void testWriteCompatibleContainer(boolean referenceTracking, boolean enab byte[] objBytes = fury.serialize(beanA); Object obj2 = newFury.deserialize(objBytes); Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(obj2, newBeanA)); - Assert.assertEquals(fury.deserialize(newFury.serialize(beanA)), beanA); } @Test(dataProvider = "config") - public void testWriteCompatibleCollection(boolean referenceTracking, boolean enableCodegen) - throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); + public void testWriteCompatibleCollection( + boolean referenceTracking, boolean scopedMetaShare, boolean enableCodegen) throws Exception { + Supplier builder = + () -> + Fury.builder() + .withLanguage(Language.JAVA) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(scopedMetaShare) + .requireClassRegistration(false); + Fury fury = builder.get().build(); CollectionFields collectionFields = UnmodifiableSerializersTest.createCollectionFields(); { Object o = serDe(fury, collectionFields); @@ -267,15 +248,7 @@ public void testWriteCompatibleCollection(boolean referenceTracking, boolean ena Class cls2 = ClassUtils.createCompatibleClass2(); Object newObj = cls2.newInstance(); ReflectionUtils.unsafeCopy(collectionFields, newObj); - Fury fury2 = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .withClassLoader(cls2.getClassLoader()) - .build(); + Fury fury2 = builder.get().withClassLoader(cls2.getClassLoader()).build(); byte[] bytes1 = fury2.serialize(newObj); Object deserialized = fury.deserialize(bytes1); Assert.assertTrue( @@ -304,16 +277,18 @@ public void testWriteCompatibleCollection(boolean referenceTracking, boolean ena } @Test(dataProvider = "config") - public void testWriteCompatibleMap(boolean referenceTracking, boolean enableCodegen) - throws Exception { - Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); + public void testWriteCompatibleMap( + boolean referenceTracking, boolean scopedMetaShare, boolean enableCodegen) throws Exception { + Supplier builder = + () -> + Fury.builder() + .withLanguage(Language.JAVA) + .withRefTracking(referenceTracking) + .withCodegen(enableCodegen) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(scopedMetaShare) + .requireClassRegistration(false); + Fury fury = builder.get().build(); MapFields mapFields = UnmodifiableSerializersTest.createMapFields(); { Object o = serDe(fury, mapFields); @@ -324,14 +299,7 @@ public void testWriteCompatibleMap(boolean referenceTracking, boolean enableCode Class cls = ClassUtils.createCompatibleClass3(); Object newObj = cls.newInstance(); ReflectionUtils.unsafeCopy(mapFields, newObj); - Fury fury2 = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(referenceTracking) - .withCodegen(enableCodegen) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); + Fury fury2 = builder.get().withClassLoader(cls.getClassLoader()).build(); byte[] bytes1 = fury2.serialize(newObj); Object deserialized = fury.deserialize(bytes1); Assert.assertTrue( diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/CompatibleSerializerTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/CompatibleSerializerTest.java index 5b6818d752..3b6222d2e8 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/CompatibleSerializerTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/CompatibleSerializerTest.java @@ -101,7 +101,7 @@ public void testWriteCompatibleBasic(boolean referenceTrackingConfig) throws Exc Object o2 = fury.deserialize(newFury.serialize(o1)); List fields = Arrays.stream(fooClass.getDeclaredFields()) - .map(f -> f.getDeclaringClass().getSimpleName() + f.getType() + f.getName()) + .map(f -> f.getDeclaringClass().getSimpleName() + f.getName()) .collect(Collectors.toList()); Assert.assertTrue(ReflectionUtils.objectFieldsEquals(new HashSet<>(fields), o2, foo)); } @@ -391,16 +391,16 @@ public static Class createCompatibleClass3() { MapFields.class, code, CompatibleSerializerTest.class + "createCompatibleClass3"); } - @Test(dataProvider = "compressNumber") - public void testCompressInt(boolean compressNumber) throws Exception { + @Test(dataProvider = "compressNumberScopedMetaShare") + public void testCompressInt(boolean compressNumber, boolean scopedMetaShare) throws Exception { + Class structClass = Struct.createNumberStructClass("CompatibleCompressIntStruct", 2); Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + builder() .withNumberCompressed(compressNumber) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) + .withClassLoader(structClass.getClassLoader()) + .withScopedMetaShare(scopedMetaShare) .build(); - Class structClass = Struct.createNumberStructClass("CompatibleCompressIntStruct", 2); serDeCheck(fury, Struct.createPOJO(structClass)); } } diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/DuplicateFieldsTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/DuplicateFieldsTest.java index ae87c01def..28ab15b7d6 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/DuplicateFieldsTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/DuplicateFieldsTest.java @@ -27,6 +27,7 @@ import org.apache.fury.FuryTestBase; import org.apache.fury.builder.CodecUtils; import org.apache.fury.config.CompatibleMode; +import org.apache.fury.config.FuryBuilder; import org.apache.fury.config.Language; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.MemoryUtils; @@ -97,21 +98,22 @@ public void testDuplicateFieldsNoCompatible() { } } - @Test() - public void testDuplicateFieldsCompatible() { + @Test(dataProvider = "scopedMetaShare") + public void testDuplicateFieldsCompatible(boolean scopedMetaShare) { C c = new C(); ((B) c).f1 = 100; c.f1 = -100; assertEquals(((B) c).f1, 100); assertEquals(c.f1, -100); - Fury fury = + FuryBuilder builder = Fury.builder() .withLanguage(Language.JAVA) .withRefTracking(false) .withCodegen(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); + .withScopedMetaShare(scopedMetaShare) + .requireClassRegistration(false); + Fury fury = builder.build(); { CompatibleSerializer serializer = new CompatibleSerializer<>(fury, C.class); MemoryBuffer buffer = MemoryUtils.buffer(32); @@ -134,15 +136,8 @@ public void testDuplicateFieldsCompatible() { } { // FallbackSerializer/CodegenSerializer will set itself to ClassResolver. - Fury fury1 = - Fury.builder() - .withLanguage(Language.JAVA) - .withRefTracking(false) - .withCodegen(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) - .build(); - C newC = (C) serDeCheckSerializer(fury1, c, "Compatible.*Codec"); + Fury fury1 = builder.build(); + C newC = serDeCheckSerializer(fury1, c, scopedMetaShare ? ".*Codec" : "(Compatible)?.*Codec"); assertEquals(newC.f1, c.f1); assertEquals(((B) newC).f1, ((B) c).f1); assertEquals(newC, c); diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java index 760272c6ee..e9b7d082e9 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/MetaSharedCompatibleTest.java @@ -32,6 +32,7 @@ import org.apache.fury.FuryTestBase; import org.apache.fury.builder.MetaSharedCodecBuilder; import org.apache.fury.config.CompatibleMode; +import org.apache.fury.config.FuryBuilder; import org.apache.fury.config.Language; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.resolver.MetaContext; @@ -50,7 +51,6 @@ * interoperability between them. */ public class MetaSharedCompatibleTest extends FuryTestBase { - public static Object serDeCheck(Fury fury, Object obj) { Object newObj = serDeMetaShared(fury, obj); Assert.assertEquals(newObj, obj); @@ -95,17 +95,22 @@ public static Object[][] config3() { .toArray(Object[][]::new); } + private static FuryBuilder furyBuilder() { + return Fury.builder() + .withLanguage(Language.JAVA) + .withMetaShare(true) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .requireClassRegistration(false) + .withScopedMetaShare(false); + } + @Test(dataProvider = "config1") public void testWrite(boolean referenceTracking, boolean compressNumber, boolean enableCodegen) { Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withNumberCompressed(compressNumber) .withRefTracking(referenceTracking) .withCodegen(enableCodegen) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); serDeCheck(fury, Foo.create()); serDeCheck(fury, BeanB.createBeanB(2)); @@ -120,14 +125,10 @@ public void testWriteCompatibleBasic( boolean enableCodegen2) throws Exception { Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); Object foo = Foo.create(); for (Class fooClass : @@ -138,14 +139,10 @@ public void testWriteCompatibleBasic( ReflectionUtils.unsafeCopy(foo, newFoo); MetaContext context = new MetaContext(); Fury newFury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(fooClass.getClassLoader()) .build(); MetaContext context1 = new MetaContext(); @@ -173,7 +170,7 @@ public void testWriteCompatibleBasic( Object o2 = fury.deserialize(newFury.serialize(o1)); List fields = Arrays.stream(fooClass.getDeclaredFields()) - .map(f -> f.getDeclaringClass().getSimpleName() + f.getType() + f.getName()) + .map(f -> f.getDeclaringClass().getSimpleName() + f.getName()) .collect(Collectors.toList()); Assert.assertTrue(ReflectionUtils.objectFieldsEquals(new HashSet<>(fields), o2, foo)); } @@ -208,11 +205,10 @@ public void testWriteCompatibleCollectionSimple() throws Exception { code, MetaSharedCompatibleTest.class + "testWriteCompatibleCollectionBasic_1"); Fury fury1 = - Fury.builder() + furyBuilder() .withCodegen(false) .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) .build(); code = @@ -234,11 +230,10 @@ public void testWriteCompatibleCollectionSimple() throws Exception { Object o2 = cls2.newInstance(); ReflectionUtils.unsafeCopy(beanA, o2); Fury fury2 = - Fury.builder() + furyBuilder() .withCodegen(false) .withMetaShare(true) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) .build(); @@ -279,14 +274,10 @@ public void testWriteCompatibleCollectionBasic( code, MetaSharedCompatibleTest.class + "testWriteCompatibleCollectionBasic_1"); Fury fury1 = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) .build(); code = @@ -308,14 +299,10 @@ public void testWriteCompatibleCollectionBasic( Object o2 = cls2.newInstance(); ReflectionUtils.unsafeCopy(beanA, o2); Fury fury2 = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen3) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) .build(); @@ -339,14 +326,10 @@ public void testWriteCompatibleCollectionBasic( Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(obj2, o2)); Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); // fury <-> fury2 is a new channel, which needs a new context. MetaContext context = new MetaContext(); @@ -364,14 +347,10 @@ public void testWriteCompatibleContainer( boolean enableCodegen2) throws Exception { Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); MetaContext context = new MetaContext(); BeanA beanA = BeanA.createBeanA(2); @@ -380,14 +359,10 @@ public void testWriteCompatibleContainer( Object newBeanA = cls.newInstance(); ReflectionUtils.unsafeCopy(beanA, newBeanA); Fury newFury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls.getClassLoader()) .build(); MetaContext context1 = new MetaContext(); @@ -422,14 +397,10 @@ public void testWriteCompatibleCollection( boolean enableCodegen2) throws Exception { Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); CollectionFields collectionFields = UnmodifiableSerializersTest.createCollectionFields(); { @@ -444,14 +415,10 @@ public void testWriteCompatibleCollection( Object newObj = cls2.newInstance(); ReflectionUtils.unsafeCopy(collectionFields, newObj); Fury fury2 = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) .build(); MetaContext context2 = new MetaContext(); @@ -499,14 +466,10 @@ public void testWriteCompatibleMap( boolean enableCodegen2) throws Exception { Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); MetaContext context = new MetaContext(); MapFields mapFields = UnmodifiableSerializersTest.createMapFields(); @@ -520,14 +483,10 @@ public void testWriteCompatibleMap( Object newObj = cls.newInstance(); ReflectionUtils.unsafeCopy(mapFields, newObj); Fury fury2 = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build(); MetaContext context2 = new MetaContext(); fury2.getSerializationContext().setMetaContext(context2); @@ -592,14 +551,10 @@ public void testDuplicateFields( + " int intField1;\n" + "}"); Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen1) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) .build(); MetaContext context = new MetaContext(); @@ -632,14 +587,10 @@ public void testDuplicateFields( + " int intField2;\n" + "}"); Fury fury2 = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen2) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls2.getClassLoader()) .build(); MetaContext context2 = new MetaContext(); @@ -691,14 +642,10 @@ void testEmptySubClass(boolean referenceTracking, boolean compressNumber, boolea + "public class DuplicateFieldsClass2 extends MetaSharedCompatibleTest.DuplicateFieldsClass1 {\n" + "}"); Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + furyBuilder() .withRefTracking(referenceTracking) .withNumberCompressed(compressNumber) .withCodegen(enableCodegen) - .withMetaShare(true) - .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .withClassLoader(cls1.getClassLoader()) .build(); Object o1 = cls1.newInstance(); diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java index d4012412a8..9fadf6834e 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/NonexistentClassSerializersTest.java @@ -20,6 +20,7 @@ package org.apache.fury.serializer; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -45,6 +46,7 @@ public class NonexistentClassSerializersTest extends FuryTestBase { public static Object[][] config() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // referenceTracking + ImmutableSet.of(true, false), // scoped meta share ImmutableSet.of(true, false), // fury1 enable codegen ImmutableSet.of(true, false) // fury2 enable codegen ) @@ -65,20 +67,24 @@ public static Object[][] metaShareConfig() { .toArray(Object[][]::new); } - private FuryBuilder furyBuilder() { + private FuryBuilder furyBuilder(boolean scoped) { return Fury.builder() .withLanguage(Language.JAVA) .withCompatibleMode(CompatibleMode.COMPATIBLE) .requireClassRegistration(false) .withCodegen(false) + .withScopedMetaShare(scoped) .withDeserializeNonexistentClass(true); } @Test(dataProvider = "config") public void testSkipNonexistent( - boolean referenceTracking, boolean enableCodegen1, boolean enableCodegen2) { + boolean referenceTracking, + boolean scopedMetaShare, + boolean enableCodegen1, + boolean enableCodegen2) { Fury fury = - furyBuilder() + furyBuilder(scopedMetaShare) .withRefTracking(referenceTracking) .withCodegen(enableCodegen1) .withCompatibleMode(CompatibleMode.COMPATIBLE) @@ -92,45 +98,32 @@ public void testSkipNonexistent( Object pojo = Struct.createPOJO(structClass); byte[] bytes = fury.serialize(pojo); Fury fury2 = - furyBuilder() + furyBuilder(scopedMetaShare) .withRefTracking(referenceTracking) .withCodegen(enableCodegen2) .withClassLoader(classLoader) .build(); Object o = fury2.deserialize(bytes); - assertEquals(o.getClass(), NonexistentClass.NonexistentSkip.class); + assertTrue(o instanceof NonexistentClass, "Unexpect type " + o.getClass()); } } - @DataProvider - public static Object[][] scopedMetaShare() { - return new Object[][] {{false}, {true}}; - } - @Test(dataProvider = "scopedMetaShare") public void testNonexistentEnum(boolean scopedMetaShare) { - FuryBuilder builder = furyBuilder(); - if (scopedMetaShare) { - builder.withMetaShare(true).withScopedMetaShare(true); - } - Fury fury = builder.withDeserializeNonexistentClass(true).build(); + Fury fury = furyBuilder(scopedMetaShare).withDeserializeNonexistentClass(true).build(); String enumCode = ("enum TestEnum {" + " A, B" + "}"); Class cls = JaninoUtils.compileClass(getClass().getClassLoader(), "", "TestEnum", enumCode); Object c = cls.getEnumConstants()[1]; assertEquals(c.toString(), "B"); byte[] bytes = fury.serialize(c); Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - Fury fury2 = builder.withDeserializeNonexistentClass(true).build(); + Fury fury2 = furyBuilder(scopedMetaShare).withDeserializeNonexistentClass(true).build(); Object o = fury2.deserialize(bytes); assertEquals(o, NonexistentClass.NonexistentEnum.V1); } @Test(dataProvider = "scopedMetaShare") public void testNonexistentEnumAndArrayField(boolean scopedMetaShare) throws Exception { - FuryBuilder builder = furyBuilder(); - if (scopedMetaShare) { - builder.withMetaShare(true).withScopedMetaShare(true); - } String enumStructCode1 = ("public class TestEnumStruct {\n" + " public enum TestEnum {\n" @@ -157,7 +150,7 @@ public void testNonexistentEnumAndArrayField(boolean scopedMetaShare) throws Exc enumArray2[1] = enumArray; ReflectionUtils.setObjectFieldValue(o, "f4", enumArray2); Fury fury1 = - builder + furyBuilder(scopedMetaShare) .withDeserializeNonexistentClass(true) .withClassLoader(cls1.getClassLoader()) .build(); @@ -174,7 +167,11 @@ public void testNonexistentEnumAndArrayField(boolean scopedMetaShare) throws Exc "", "TestEnumStruct", ("public class TestEnumStruct {" + " public String f1;" + "}"))); - Fury fury2 = builder.withDeserializeNonexistentClass(true).withClassLoader(classLoader).build(); + Fury fury2 = + furyBuilder(scopedMetaShare) + .withDeserializeNonexistentClass(true) + .withClassLoader(classLoader) + .build(); Object o1 = fury2.deserialize(bytes); Assert.assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f1"), "str"); } @@ -215,7 +212,7 @@ public void testSkipNonexistentObjectArrayField(boolean componentFinal) throws E arr2D[1] = arr; ReflectionUtils.setObjectFieldValue(o, "f4", arr2D); Fury fury1 = - furyBuilder() + furyBuilder(false) .withDeserializeNonexistentClass(true) .withClassLoader(cls1.getClassLoader()) .build(); @@ -233,7 +230,10 @@ public void testSkipNonexistentObjectArrayField(boolean componentFinal) throws E "TestArrayStruct", ("public class TestArrayStruct {" + " public String f1;" + "}"))); Fury fury2 = - furyBuilder().withDeserializeNonexistentClass(true).withClassLoader(classLoader).build(); + furyBuilder(false) + .withDeserializeNonexistentClass(true) + .withClassLoader(classLoader) + .build(); Object o1 = fury2.deserialize(bytes); Assert.assertEquals(ReflectionUtils.getObjectFieldValue(o1, "f1"), "str"); } @@ -245,7 +245,7 @@ public void testDeserializeNonexistentNewFury( boolean enableCodegen2, boolean enableCodegen3) { Fury fury = - furyBuilder() + furyBuilder(false) .withRefTracking(referenceTracking) .withCodegen(enableCodegen1) .withMetaShare(true) @@ -261,7 +261,7 @@ public void testDeserializeNonexistentNewFury( fury.getSerializationContext().setMetaContext(context1); byte[] bytes = fury.serialize(pojo); Fury fury2 = - furyBuilder() + furyBuilder(false) .withRefTracking(referenceTracking) .withCodegen(enableCodegen2) .withMetaShare(true) @@ -274,7 +274,7 @@ public void testDeserializeNonexistentNewFury( fury2.getSerializationContext().setMetaContext(context2); byte[] bytes2 = fury2.serialize(o2); Fury fury3 = - furyBuilder() + furyBuilder(false) .withRefTracking(referenceTracking) .withCodegen(enableCodegen3) .withMetaShare(true) @@ -295,7 +295,7 @@ public void testDeserializeNonexistent( boolean enableCodegen2, boolean enableCodegen3) { Fury fury = - furyBuilder() + furyBuilder(false) .withRefTracking(referenceTracking) .withCodegen(enableCodegen1) .withMetaShare(true) @@ -310,14 +310,14 @@ public void testDeserializeNonexistent( Struct.createStructClass("TestSkipNonexistentClass3", 2) }) { Fury fury2 = - furyBuilder() + furyBuilder(false) .withRefTracking(referenceTracking) .withCodegen(enableCodegen2) .withMetaShare(true) .withClassLoader(classLoader) .build(); Fury fury3 = - furyBuilder() + furyBuilder(false) .withRefTracking(referenceTracking) .withCodegen(enableCodegen3) .withMetaShare(true) @@ -342,14 +342,17 @@ public void testDeserializeNonexistent( } } - @Test - public void testThrowExceptionIfClassNotExist() { - Fury fury = furyBuilder().withDeserializeNonexistentClass(false).build(); + @Test(dataProvider = "scopedMetaShare") + public void testThrowExceptionIfClassNotExist(boolean scopedMetaShare) { + Fury fury = furyBuilder(scopedMetaShare).withDeserializeNonexistentClass(false).build(); ClassLoader classLoader = getClass().getClassLoader(); Class structClass = Struct.createNumberStructClass("TestSkipNonexistentClass1", 2); Object pojo = Struct.createPOJO(structClass); Fury fury2 = - furyBuilder().withDeserializeNonexistentClass(false).withClassLoader(classLoader).build(); + furyBuilder(scopedMetaShare) + .withDeserializeNonexistentClass(false) + .withClassLoader(classLoader) + .build(); byte[] bytes = fury.serialize(pojo); Assert.assertThrows(RuntimeException.class, () -> fury2.deserialize(bytes)); } diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java index 09f5646e18..38b0a1144d 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/ProtocolInteroperabilityTest.java @@ -61,6 +61,7 @@ public static Object[][] fury() { return Sets.cartesianProduct( ImmutableSet.of(true, false), // referenceTracking ImmutableSet.of(true, false), // compressNumber + ImmutableSet.of(true, false), // scopedMetaShare ImmutableSet.of(CompatibleMode.SCHEMA_CONSISTENT, CompatibleMode.COMPATIBLE)) .stream() .map(List::toArray) @@ -72,7 +73,8 @@ public static Object[][] fury() { .withRefTracking((Boolean) c[0]) .withNumberCompressed((Boolean) c[1]) .withCodegen(false) - .withCompatibleMode((CompatibleMode) c[2]) + .withScopedMetaShare((Boolean) c[2]) + .withCompatibleMode((CompatibleMode) c[3]) .requireClassRegistration(false) .build(), Fury.builder() @@ -80,7 +82,8 @@ public static Object[][] fury() { .withRefTracking((Boolean) c[0]) .withNumberCompressed((Boolean) c[1]) .withCodegen(true) - .withCompatibleMode((CompatibleMode) c[2]) + .withScopedMetaShare((Boolean) c[2]) + .withCompatibleMode((CompatibleMode) c[3]) .requireClassRegistration(false) .build() }) @@ -237,6 +240,7 @@ public static Object[][] metaShareFury() { .withLanguage(Language.JAVA) .withMetaShare(true) .withCompatibleMode((CompatibleMode) c[0]) + .withScopedMetaShare(false) .withCodegen(false) .requireClassRegistration(false) .build(), @@ -244,6 +248,7 @@ public static Object[][] metaShareFury() { .withLanguage(Language.JAVA) .withMetaShare(true) .withCompatibleMode((CompatibleMode) c[0]) + .withScopedMetaShare(false) .withCodegen(true) .requireClassRegistration(false) .build() diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/ScopedMetaShareTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/ScopedMetaShareTest.java new file mode 100644 index 0000000000..2adedddbda --- /dev/null +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/ScopedMetaShareTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.serializer; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.apache.fury.Fury; +import org.apache.fury.FuryTestBase; +import org.apache.fury.config.CompatibleMode; +import org.apache.fury.config.FuryBuilder; +import org.apache.fury.reflect.ReflectionUtils; +import org.apache.fury.test.bean.Foo; +import org.testng.Assert; +import org.testng.annotations.Test; + +// Most scoped meta share tests are located in CompatibleTest module. +public class ScopedMetaShareTest extends FuryTestBase { + + // Test registration doesn't skip write class meta + @Test + public void testRegister() throws Exception { + Supplier builder = + () -> + builder() + .withCodegen(true) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withScopedMetaShare(true); + Object foo = Foo.create(); + Class fooClass = Foo.createCompatibleClass1(); + Object newFoo = fooClass.newInstance(); + ReflectionUtils.unsafeCopy(foo, newFoo); + Fury fury = builder.get().build(); + fury.register(Foo.class); + Fury newFury = builder.get().withClassLoader(fooClass.getClassLoader()).build(); + newFury.register(fooClass); + { + byte[] foo1Bytes = newFury.serialize(newFoo); + Object deserialized = fury.deserialize(foo1Bytes); + Assert.assertEquals(deserialized.getClass(), Foo.class); + Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(deserialized, newFoo)); + byte[] fooBytes = fury.serialize(deserialized); + Assert.assertTrue(ReflectionUtils.objectFieldsEquals(newFury.deserialize(fooBytes), newFoo)); + } + { + byte[] bytes1 = fury.serialize(foo); + Object o1 = newFury.deserialize(bytes1); + Assert.assertTrue(ReflectionUtils.objectCommonFieldsEquals(o1, foo)); + Object o2 = fury.deserialize(newFury.serialize(o1)); + List fields = + Arrays.stream(fooClass.getDeclaredFields()) + .map(f -> f.getDeclaringClass().getSimpleName() + f.getName()) + .collect(Collectors.toList()); + Assert.assertTrue(ReflectionUtils.objectFieldsEquals(new HashSet<>(fields), o2, foo)); + } + { + Object o3 = fury.deserialize(newFury.serialize(foo)); + Assert.assertTrue(ReflectionUtils.objectFieldsEquals(o3, foo)); + } + } +} diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java index 57e906bb10..4d118442ee 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/ChildContainerSerializersTest.java @@ -35,7 +35,6 @@ import org.apache.fury.Fury; import org.apache.fury.FuryTestBase; import org.apache.fury.config.CompatibleMode; -import org.apache.fury.config.Language; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -63,19 +62,23 @@ public static class ChildHashSet extends HashSet {} public static Object[][] furyConfig() { return new Object[][] { { - Fury.builder() - .withLanguage(Language.JAVA) + builder() .withRefTracking(false) + .withScopedMetaShare(false) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .requireClassRegistration(false) .build() }, { - Fury.builder() - .withLanguage(Language.JAVA) + builder() + .withRefTracking(false) + .withScopedMetaShare(true) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build() + }, + { + builder() .withRefTracking(false) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .requireClassRegistration(false) .build() }, }; @@ -176,12 +179,11 @@ public void testSerializeCustomPrivateMap(boolean enableCodegen) { UserDO outerDO = new UserDO(); outerDO.setFeatures(features); Fury fury = - Fury.builder() - .withLanguage(Language.JAVA) + builder() .withCompatibleMode(CompatibleMode.COMPATIBLE) .withDeserializeNonexistentClass(true) .withMetaShare(true) - .requireClassRegistration(false) + .withScopedMetaShare(false) .withCodegen(enableCodegen) .build(); serDeMetaShared(fury, outerDO);