From 4ed36ac16a059f66e00d137a3b64685e647b4f31 Mon Sep 17 00:00:00 2001 From: LiangliangSui Date: Mon, 29 Jul 2024 22:27:24 +0800 Subject: [PATCH 1/9] update Signed-off-by: LiangliangSui --- .../src/main/java/org/apache/fury/Fury.java | 17 +++++++++++++---- .../org/apache/fury/collection/ObjectArray.java | 4 ++++ .../org/apache/fury/resolver/ClassResolver.java | 1 + .../src/test/java/org/apache/fury/FuryTest.java | 17 +++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index e430a42638..e8a026ffd2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -48,6 +48,7 @@ import org.apache.fury.resolver.ClassInfoHolder; import org.apache.fury.resolver.ClassResolver; import org.apache.fury.resolver.MapRefResolver; +import org.apache.fury.resolver.MetaContext; import org.apache.fury.resolver.MetaStringResolver; import org.apache.fury.resolver.NoRefResolver; import org.apache.fury.resolver.RefResolver; @@ -339,12 +340,14 @@ private void write(MemoryBuffer buffer, Object obj) { buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. } // reduce caller stack - if (!refResolver.writeRefOrNull(buffer, obj)) { + boolean isNull = refResolver.writeRefOrNull(buffer, obj); + if (!isNull) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); classResolver.writeClass(buffer, classInfo); writeData(buffer, classInfo, obj); } - if (shareMeta) { + MetaContext metaContext = getSerializationContext().getMetaContext(); + if (shareMeta && !isNull && metaContext != null && !metaContext.writingClassDefs.isEmpty()) { buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); classResolver.writeClassDefs(buffer); } @@ -1063,9 +1066,12 @@ public void serializeJavaObject(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); writeData(buffer, classInfo, obj); + MetaContext metaContext = getSerializationContext().getMetaContext(); + if (metaContext != null && !metaContext.writingClassDefs.isEmpty()) { + buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); + classResolver.writeClassDefs(buffer); + } } - buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); - classResolver.writeClassDefs(buffer); } else { if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); @@ -1437,6 +1443,9 @@ private void serializeToStream(OutputStream outputStream, Consumer private void readClassDefs(MemoryBuffer buffer) { int relativeClassDefOffset = buffer.readInt32(); + if (relativeClassDefOffset == -1) { + return; + } int readerIndex = buffer.readerIndex(); buffer.readerIndex(readerIndex + relativeClassDefOffset); classResolver.readClassDefs(buffer); diff --git a/java/fury-core/src/main/java/org/apache/fury/collection/ObjectArray.java b/java/fury-core/src/main/java/org/apache/fury/collection/ObjectArray.java index 94c08865f8..9f100e0e00 100644 --- a/java/fury-core/src/main/java/org/apache/fury/collection/ObjectArray.java +++ b/java/fury-core/src/main/java/org/apache/fury/collection/ObjectArray.java @@ -90,6 +90,10 @@ public int size() { return size; } + public boolean isEmpty() { + return size == 0; + } + /** * Set all object array elements to null. This method is faster than {@link Arrays#fill} for large * arrays (> 128). diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index 883a63d0a0..186a60df18 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -1470,6 +1470,7 @@ private void writeClassDefs( */ public void readClassDefs(MemoryBuffer buffer) { MetaContext metaContext = fury.getSerializationContext().getMetaContext(); + assert metaContext != null : SET_META__CONTEXT_MSG; int numClassDefs = buffer.readVarUint32Small14(); for (int i = 0; i < numClassDefs; i++) { long id = buffer.readInt64(); diff --git a/java/fury-core/src/test/java/org/apache/fury/FuryTest.java b/java/fury-core/src/test/java/org/apache/fury/FuryTest.java index 032c609e3c..1e4b5a00ae 100644 --- a/java/fury-core/src/test/java/org/apache/fury/FuryTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/FuryTest.java @@ -62,6 +62,7 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.MemoryUtils; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.MetaContext; import org.apache.fury.serializer.ArraySerializersTest; import org.apache.fury.serializer.EnumSerializerTest; import org.apache.fury.serializer.ObjectSerializer; @@ -598,4 +599,20 @@ public void testPrintReadObjectsWhenFailed() { Assert.assertTrue(e.getMessage().contains("[a, b]")); } } + + @Test + public void testNullObjSerAndDe() { + Fury fury = + Fury.builder() + .withRefTracking(true) + .requireClassRegistration(false) + .withMetaShare(true) + .build(); + MetaContext metaContext = new MetaContext(); + fury.getSerializationContext().setMetaContext(metaContext); + byte[] bytes = fury.serializeJavaObjectAndClass(null); + fury.getSerializationContext().setMetaContext(metaContext); + Object obj = fury.deserializeJavaObjectAndClass(bytes); + assertNull(obj); + } } From 852d86dea6ef0bba20305eee213c5f29bf96197d Mon Sep 17 00:00:00 2001 From: LiangliangSui Date: Mon, 29 Jul 2024 22:55:42 +0800 Subject: [PATCH 2/9] update Signed-off-by: LiangliangSui --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index e8a026ffd2..061d913ea7 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -1444,6 +1444,7 @@ private void serializeToStream(OutputStream outputStream, Consumer private void readClassDefs(MemoryBuffer buffer) { int relativeClassDefOffset = buffer.readInt32(); if (relativeClassDefOffset == -1) { + classDefEndOffset = buffer.readerIndex(); return; } int readerIndex = buffer.readerIndex(); From c04d85b207d94314cdd29275b327c5aeaaa1afee Mon Sep 17 00:00:00 2001 From: LiangliangSui Date: Mon, 29 Jul 2024 23:17:37 +0800 Subject: [PATCH 3/9] update Signed-off-by: LiangliangSui --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 061d913ea7..9faa747d78 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -156,6 +156,7 @@ public Fury(FuryBuilder builder, ClassLoader classLoader) { arrayListSerializer = new ArrayListSerializer(this); hashMapSerializer = new HashMapSerializer(this); originToCopyMap = new IdentityMap<>(); + classDefEndOffset = -1; LOG.info("Created new fury {}", this); } @@ -795,7 +796,7 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (shareMeta) { + if (shareMeta && classDefEndOffset != -1) { buffer.readerIndex(classDefEndOffset); } resetRead(); @@ -1122,7 +1123,7 @@ public T deserializeJavaObject(MemoryBuffer buffer, Class cls) { } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (shareMeta) { + if (shareMeta && classDefEndOffset != -1) { buffer.readerIndex(classDefEndOffset); } resetRead(); @@ -1232,7 +1233,7 @@ public Object deserializeJavaObjectAndClass(MemoryBuffer buffer) { } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (shareMeta) { + if (shareMeta && classDefEndOffset != -1) { buffer.readerIndex(classDefEndOffset); } resetRead(); @@ -1444,7 +1445,6 @@ private void serializeToStream(OutputStream outputStream, Consumer private void readClassDefs(MemoryBuffer buffer) { int relativeClassDefOffset = buffer.readInt32(); if (relativeClassDefOffset == -1) { - classDefEndOffset = buffer.readerIndex(); return; } int readerIndex = buffer.readerIndex(); @@ -1484,6 +1484,7 @@ public void resetRead() { nativeObjects.clear(); peerOutOfBandEnabled = false; depth = 0; + classDefEndOffset = -1; } public void resetCopy() { From 2e7d69c6feab566aa428cb309b0941b6cc04ec7c Mon Sep 17 00:00:00 2001 From: LiangliangSui Date: Wed, 31 Jul 2024 19:48:07 +0800 Subject: [PATCH 4/9] update Signed-off-by: LiangliangSui --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 9faa747d78..ed1b4a67ac 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -341,14 +341,13 @@ private void write(MemoryBuffer buffer, Object obj) { buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. } // reduce caller stack - boolean isNull = refResolver.writeRefOrNull(buffer, obj); - if (!isNull) { + if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); classResolver.writeClass(buffer, classInfo); writeData(buffer, classInfo, obj); } MetaContext metaContext = getSerializationContext().getMetaContext(); - if (shareMeta && !isNull && metaContext != null && !metaContext.writingClassDefs.isEmpty()) { + if (shareMeta && metaContext != null && !metaContext.writingClassDefs.isEmpty()) { buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); classResolver.writeClassDefs(buffer); } From 1cb6b9a51889e1eb29d85608cc2d663714c43b63 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 5 Aug 2024 10:09:42 +0800 Subject: [PATCH 5/9] Update java/fury-core/src/main/java/org/apache/fury/Fury.java --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index ed1b4a67ac..d8e89c0379 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -346,7 +346,7 @@ private void write(MemoryBuffer buffer, Object obj) { classResolver.writeClass(buffer, classInfo); writeData(buffer, classInfo, obj); } - MetaContext metaContext = getSerializationContext().getMetaContext(); + MetaContext metaContext = serializationContext.getMetaContext(); if (shareMeta && metaContext != null && !metaContext.writingClassDefs.isEmpty()) { buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); classResolver.writeClassDefs(buffer); From 73f6e881ff184cbeae2a7a5fcbcd74c90e2597fa Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 5 Aug 2024 10:13:27 +0800 Subject: [PATCH 6/9] Update java/fury-core/src/main/java/org/apache/fury/Fury.java --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index d8e89c0379..76c5e67a26 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -1232,7 +1232,7 @@ public Object deserializeJavaObjectAndClass(MemoryBuffer buffer) { } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (shareMeta && classDefEndOffset != -1) { + if (classDefEndOffset != -1) { buffer.readerIndex(classDefEndOffset); } resetRead(); From ac3b8e030c7e22376b98db27e2f57b559d39372e Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 5 Aug 2024 10:13:47 +0800 Subject: [PATCH 7/9] Update java/fury-core/src/main/java/org/apache/fury/Fury.java --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 76c5e67a26..6bd4b1910b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -1122,7 +1122,7 @@ public T deserializeJavaObject(MemoryBuffer buffer, Class cls) { } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (shareMeta && classDefEndOffset != -1) { + if (classDefEndOffset != -1) { buffer.readerIndex(classDefEndOffset); } resetRead(); From b900195579ebeae3c97a5477b4be65f7ccd62527 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 5 Aug 2024 10:14:31 +0800 Subject: [PATCH 8/9] Update java/fury-core/src/main/java/org/apache/fury/Fury.java --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 6bd4b1910b..b9d830c3ee 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -795,7 +795,7 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB } catch (Throwable t) { throw ExceptionUtils.handleReadFailed(this, t); } finally { - if (shareMeta && classDefEndOffset != -1) { + if (classDefEndOffset != -1) { buffer.readerIndex(classDefEndOffset); } resetRead(); From 498df600296fa490fd374c9005e8128ba8b8d3a8 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 5 Aug 2024 10:14:38 +0800 Subject: [PATCH 9/9] Update java/fury-core/src/main/java/org/apache/fury/Fury.java --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index b9d830c3ee..787d47beae 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -1066,7 +1066,7 @@ public void serializeJavaObject(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); writeData(buffer, classInfo, obj); - MetaContext metaContext = getSerializationContext().getMetaContext(); + MetaContext metaContext = serializationContext.getMetaContext(); if (metaContext != null && !metaContext.writingClassDefs.isEmpty()) { buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); classResolver.writeClassDefs(buffer);