Skip to content

Commit

Permalink
fix(java): fix fastutil Object2ObjectOpenHashMap serialization (#1618)
Browse files Browse the repository at this point in the history
## What does this PR do?

fix fastutil Object2ObjectOpenHashMap serialization

## Related issues

Closes #1615 


## Does this PR introduce any user-facing change?

<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/incubator-fury/issues/new/choose)
describing the need to do so and update the document if necessary.
-->

- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?


## Benchmark

<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
  • Loading branch information
chaokunyang authored May 9, 2024
1 parent b2fdd68 commit 06a74a1
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@
import org.apache.fury.reflect.ReflectionUtils;
import org.apache.fury.reflect.TypeRef;
import org.apache.fury.serializer.PrimitiveSerializers;
import org.apache.fury.serializer.collection.CollectionSerializer;
import org.apache.fury.serializer.collection.MapSerializer;
import org.apache.fury.serializer.collection.AbstractCollectionSerializer;
import org.apache.fury.serializer.collection.AbstractMapSerializer;
import org.apache.fury.type.Descriptor;
import org.apache.fury.type.TypeUtils;
import org.apache.fury.util.MurmurHash3;
Expand Down Expand Up @@ -567,7 +567,8 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType) {
if (fieldType == FieldTypes.COLLECTION_ELEMENT_FINAL) {
ClassInfo elementClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, classInfoHolder);
CollectionSerializer collectionSerializer = (CollectionSerializer) classInfo.getSerializer();
AbstractCollectionSerializer collectionSerializer =
(AbstractCollectionSerializer) classInfo.getSerializer();
try {
collectionSerializer.setElementSerializer(elementClassInfo.getSerializer());
o = collectionSerializer.read(buffer);
Expand All @@ -580,7 +581,7 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType) {
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, classInfoHolder);
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
Expand All @@ -594,7 +595,7 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType) {
} else if (fieldType == FieldTypes.MAP_KEY_FINAL) {
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, classInfoHolder);
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
o = mapSerializer.read(buffer);
Expand All @@ -607,7 +608,7 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType) {
Preconditions.checkArgument(fieldType == FieldTypes.MAP_VALUE_FINAL);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, classInfoHolder);
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
o = mapSerializer.read(buffer);
Expand All @@ -625,7 +626,8 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType, FieldInf
if (fieldType == FieldTypes.COLLECTION_ELEMENT_FINAL) {
ClassInfo elementClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, fieldInfo.getClassInfoHolder());
CollectionSerializer collectionSerializer = (CollectionSerializer) classInfo.getSerializer();
AbstractCollectionSerializer collectionSerializer =
(AbstractCollectionSerializer) classInfo.getSerializer();
try {
collectionSerializer.setElementSerializer(elementClassInfo.getSerializer());
o = collectionSerializer.read(buffer);
Expand All @@ -638,7 +640,7 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType, FieldInf
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, fieldInfo.getClassInfoHolder());
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
Expand All @@ -652,7 +654,7 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType, FieldInf
} else if (fieldType == FieldTypes.MAP_KEY_FINAL) {
ClassInfo keyClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, fieldInfo.getClassInfoHolder());
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
o = mapSerializer.read(buffer);
Expand All @@ -665,7 +667,7 @@ private Object readObjectWithFinal(MemoryBuffer buffer, byte fieldType, FieldInf
Preconditions.checkArgument(fieldType == FieldTypes.MAP_VALUE_FINAL);
ClassInfo valueClassInfo = classResolver.readClassInfo(buffer, classInfoHolder);
ClassInfo classInfo = classResolver.readClassInfo(buffer, fieldInfo.getClassInfoHolder());
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
o = mapSerializer.read(buffer);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
import org.apache.fury.resolver.ClassResolver;
import org.apache.fury.resolver.FieldResolver;
import org.apache.fury.resolver.RefResolver;
import org.apache.fury.serializer.collection.CollectionSerializer;
import org.apache.fury.serializer.collection.MapSerializer;
import org.apache.fury.serializer.collection.AbstractCollectionSerializer;
import org.apache.fury.serializer.collection.AbstractMapSerializer;
import org.apache.fury.util.Preconditions;
import org.apache.fury.util.record.RecordInfo;
import org.apache.fury.util.record.RecordUtils;
Expand Down Expand Up @@ -241,7 +241,8 @@ private void writeCollectionField(
// following write is consistent with `BaseSeqCodecBuilder.serializeForCollection`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
CollectionSerializer collectionSerializer = (CollectionSerializer) classInfo.getSerializer();
AbstractCollectionSerializer collectionSerializer =
(AbstractCollectionSerializer) classInfo.getSerializer();
try {
collectionSerializer.setElementSerializer(elementClassInfo.getSerializer());
collectionSerializer.write(buffer, fieldValue);
Expand All @@ -261,7 +262,7 @@ private void writeMapKVFinal(
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
Expand All @@ -281,7 +282,7 @@ private void writeMapKeyFinal(
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setKeySerializer(keyClassInfo.getSerializer());
mapSerializer.write(buffer, fieldValue);
Expand All @@ -299,7 +300,7 @@ private void writeMapValueFinal(
// following write is consistent with `BaseSeqCodecBuilder.serializeForMap`
ClassInfo classInfo = fieldInfo.getClassInfo(fieldValue.getClass());
classResolver.writeClass(buffer, classInfo);
MapSerializer mapSerializer = (MapSerializer) classInfo.getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer) classInfo.getSerializer();
try {
mapSerializer.setValueSerializer(valueClassInfo.getSerializer());
mapSerializer.write(buffer, fieldValue);
Expand Down
6 changes: 6 additions & 0 deletions java/fury-testsuite/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@
<version>2.57</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.5.6</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
65 changes: 65 additions & 0 deletions java/fury-testsuite/src/test/java/org/apache/fury/TestBase.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.fury;

import org.apache.fury.config.FuryBuilder;
import org.apache.fury.config.Language;
import org.testng.Assert;
import org.testng.annotations.DataProvider;

/** Fury unit test base class. */
@SuppressWarnings("unchecked")
public abstract class TestBase {

public static FuryBuilder builder() {
return Fury.builder().withLanguage(Language.JAVA).requireClassRegistration(false);
}

@DataProvider
public static Object[][] trackingRef() {
return new Object[][] {{false}, {true}};
}

@DataProvider
public static Object[][] enableCodegen() {
return new Object[][] {{false}, {true}};
}

@DataProvider
public static Object[][] compressNumber() {
return new Object[][] {{false}, {true}};
}

public static Object serDeCheck(Fury fury, Object obj) {
Object o = serDe(fury, obj);
Assert.assertEquals(o, obj);
return o;
}

public static <T> T serDe(Fury fury, T obj) {
try {
byte[] bytes = fury.serialize(obj);
return (T) (fury.deserialize(bytes));
} catch (Throwable t) {
// Catch for add breakpoint for debugging.
throw t;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.fury.test;

import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import java.util.Map;
import lombok.Data;
import org.apache.fury.Fury;
import org.apache.fury.TestBase;
import org.apache.fury.config.CompatibleMode;
import org.testng.Assert;
import org.testng.annotations.Test;

public class Object2ObjectOpenHashMapTest extends TestBase {
@Data
public static class TestObject2ObjectOpenHashMap {
Map<String, String> ext = new Object2ObjectOpenHashMap<>();
}

@Test(dataProvider = "enableCodegen")
public void testObject2ObjectOpenHashMap(boolean enableCodegen) {
Fury fury =
builder().withCompatibleMode(CompatibleMode.COMPATIBLE).withCodegen(enableCodegen).build();

TestObject2ObjectOpenHashMap o = new TestObject2ObjectOpenHashMap();
byte[] bytes = fury.serializeJavaObject(o);
Assert.assertEquals(fury.deserializeJavaObject(bytes, TestObject2ObjectOpenHashMap.class), o);
}
}

0 comments on commit 06a74a1

Please sign in to comment.