From e99b46f4b83f79a23cf240466479da9e3657810d Mon Sep 17 00:00:00 2001 From: weipeng Date: Thu, 8 Aug 2024 16:45:31 +0800 Subject: [PATCH] feat(Rust): Support polymorphism (#1795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What does this PR do? 1. Support polymorphism Add a class resolver which is used for managing the relationship between type_id and serializer. The serializer is generated by a generic function which will be called by dynamic dispatch. When a struct uses `Box` as a field type and does not specify the specific type, we will use the class resolver to load the handler by type_id. In this situation, there is some performance overhead due to hash lookup, but it greatly improves convenience. Use as follow: ```Rust #[test] fn any() { #[derive(Fury, Debug)] struct Animal { f3: String, } #[derive(Fury, Debug)] struct Person { f1: Box, } let person = Person { f1: Box::new(Animal { f3: String::from("hello"), }), }; let mut fury = Fury::default(); fury.register::(999); fury.register::(1000); let bin = fury.serialize(&person); let obj: Person = fury.deserialize(&bin).expect(""); assert_eq!(true, obj.f1.is::()) } ``` 2. Add a register function for user to register Struct and id 3. Remove tag and hash which were generate by macro before and were removed in our protocol now. ## TODO 1. Internal types like String、Set and Map should be registered by fury by default and lookup by pattern match to avoid hash overhead. 2. More unit testcases. 3. Support `Box ` --- rust/fury-core/src/buffer.rs | 11 ++ rust/fury-core/src/error.rs | 13 +- rust/fury-core/src/fury.rs | 14 ++- rust/fury-core/src/meta/type_meta.rs | 21 ++-- rust/fury-core/src/resolver/class_resolver.rs | 116 ++++++++++++++++++ rust/fury-core/src/resolver/context.rs | 4 +- rust/fury-core/src/resolver/meta_resolver.rs | 11 +- rust/fury-core/src/resolver/mod.rs | 1 + rust/fury-core/src/serializer/any.rs | 87 +++++++++++++ rust/fury-core/src/serializer/bool.rs | 5 +- rust/fury-core/src/serializer/datetime.rs | 9 +- rust/fury-core/src/serializer/list.rs | 5 +- rust/fury-core/src/serializer/map.rs | 5 +- rust/fury-core/src/serializer/mod.rs | 25 ++-- rust/fury-core/src/serializer/number.rs | 11 +- rust/fury-core/src/serializer/option.rs | 19 +-- .../src/serializer/primitive_list.rs | 31 +++-- rust/fury-core/src/serializer/set.rs | 5 +- rust/fury-core/src/serializer/string.rs | 5 +- rust/fury-core/src/types.rs | 6 +- rust/fury-derive/src/lib.rs | 16 +-- rust/fury-derive/src/object/misc.rs | 37 ++---- rust/fury-derive/src/object/read.rs | 85 ++++++++----- rust/fury-derive/src/object/serializer.rs | 10 +- rust/fury-derive/src/object/write.rs | 16 +-- rust/tests/tests/test_complex_struct.rs | 40 +++++- 26 files changed, 437 insertions(+), 171 deletions(-) create mode 100644 rust/fury-core/src/resolver/class_resolver.rs create mode 100644 rust/fury-core/src/serializer/any.rs diff --git a/rust/fury-core/src/buffer.rs b/rust/fury-core/src/buffer.rs index c81ec17fb8..5668034ca7 100644 --- a/rust/fury-core/src/buffer.rs +++ b/rust/fury-core/src/buffer.rs @@ -247,4 +247,15 @@ impl<'bf> Reader<'bf> { self.move_next(len); result } + + pub fn reset_cursor_to_here(&self) -> impl FnOnce(&mut Self) { + let raw_cursor = self.cursor; + move |this: &mut Self| { + this.cursor = raw_cursor; + } + } + + pub fn aligned(&self) -> bool { + unsafe { (self.bf.as_ptr().add(self.cursor) as usize) % std::mem::align_of::() == 0 } + } } diff --git a/rust/fury-core/src/error.rs b/rust/fury-core/src/error.rs index a6fc9967a5..e2b2e25142 100644 --- a/rust/fury-core/src/error.rs +++ b/rust/fury-core/src/error.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use super::types::{FieldType, Language}; +use super::types::Language; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -31,8 +31,8 @@ pub enum Error { #[error("BadRefFlag")] BadRefFlag, - #[error("Bad FieldType; expected: {expected:?}, actual: {actial:?}")] - FieldType { expected: FieldType, actial: i16 }, + #[error("Bad FieldType; expected: {expected:?}, actual: {actual:?}")] + FieldType { expected: i16, actual: i16 }, #[error("Bad timestamp; out-of-range number of milliseconds")] NaiveDateTime, @@ -40,8 +40,8 @@ pub enum Error { #[error("Bad date; out-of-range")] NaiveDate, - #[error("Schema is not consistent; expected: {expected:?}, actual: {actial:?}")] - StructHash { expected: u32, actial: u32 }, + #[error("Schema is not consistent; expected: {expected:?}, actual: {actual:?}")] + StructHash { expected: u32, actual: u32 }, #[error("Bad Tag Type: {0}")] TagType(u8), @@ -75,4 +75,7 @@ pub enum Error { #[error("Invalid character value for LOWER_UPPER_DIGIT_SPECIAL decoding: {value:?}")] InvalidLowerUpperDigitSpecialValue { value: u8 }, + + #[error("Unregistered type when serializing or deserializing object of Any type: {value:?}")] + UnregisteredType { value: u32 }, } diff --git a/rust/fury-core/src/fury.rs b/rust/fury-core/src/fury.rs index 376374c901..ecf3c404dc 100644 --- a/rust/fury-core/src/fury.rs +++ b/rust/fury-core/src/fury.rs @@ -17,19 +17,22 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; +use crate::resolver::class_resolver::{ClassInfo, ClassResolver}; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; -use crate::serializer::Serializer; +use crate::serializer::{Serializer, StructSerializer}; use crate::types::{config_flags, Language, Mode, SIZE_OF_REF_AND_TYPE}; pub struct Fury { mode: Mode, + class_resolver: ClassResolver, } impl Default for Fury { fn default() -> Self { Fury { mode: Mode::SchemaConsistent, + class_resolver: ClassResolver::default(), } } } @@ -82,4 +85,13 @@ impl Fury { } writer.dump() } + + pub fn get_class_resolver(&self) -> &ClassResolver { + &self.class_resolver + } + + pub fn register(&mut self, id: u32) { + let class_info = ClassInfo::new::(self, id); + self.class_resolver.register::(class_info, id); + } } diff --git a/rust/fury-core/src/meta/type_meta.rs b/rust/fury-core/src/meta/type_meta.rs index f7f781dfa7..c1086e0f25 100644 --- a/rust/fury-core/src/meta/type_meta.rs +++ b/rust/fury-core/src/meta/type_meta.rs @@ -19,20 +19,17 @@ use super::meta_string::MetaStringEncoder; use crate::buffer::{Reader, Writer}; use crate::error::Error; use crate::meta::{Encoding, MetaStringDecoder}; -use crate::types::FieldType; -//todo backward/forward compatibility -#[allow(dead_code)] pub struct FieldInfo { field_name: String, - field_type: FieldType, + field_id: i16, } impl FieldInfo { - pub fn new(field_name: &str, field_type: FieldType) -> FieldInfo { + pub fn new(field_name: &str, field_type: i16) -> FieldInfo { FieldInfo { field_name: field_name.to_string(), - field_type, + field_id: field_type, } } @@ -60,7 +57,7 @@ impl FieldInfo { .unwrap(); FieldInfo { field_name, - field_type: FieldType::try_from(type_id).unwrap(), + field_id: type_id, } } @@ -80,7 +77,7 @@ impl FieldInfo { header |= (size << 5) as u8; writer.u8(header); } - writer.i16(self.field_type as i16); + writer.i16(self.field_id); writer.bytes(encoded); Ok(writer.dump()) } @@ -99,6 +96,10 @@ impl TypeMetaLayer { } } + pub fn get_type_id(&self) -> u32 { + self.type_id + } + pub fn get_field_info(&self) -> &Vec { &self.field_info } @@ -133,6 +134,10 @@ impl TypeMeta { self.layers.first().unwrap().get_field_info() } + pub fn get_type_id(&self) -> u32 { + self.layers.first().unwrap().get_type_id() + } + pub fn from_fields(type_id: u32, field_info: Vec) -> TypeMeta { TypeMeta { hash: 0, diff --git a/rust/fury-core/src/resolver/class_resolver.rs b/rust/fury-core/src/resolver/class_resolver.rs new file mode 100644 index 0000000000..544da22d94 --- /dev/null +++ b/rust/fury-core/src/resolver/class_resolver.rs @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::context::{ReadContext, WriteContext}; +use crate::error::Error; +use crate::fury::Fury; +use crate::serializer::StructSerializer; +use std::any::TypeId; +use std::{any::Any, collections::HashMap}; + +pub struct Harness { + serializer: fn(&dyn Any, &mut WriteContext), + deserializer: fn(&mut ReadContext) -> Result, Error>, +} + +impl Harness { + pub fn new( + serializer: fn(&dyn Any, &mut WriteContext), + deserializer: fn(&mut ReadContext) -> Result, Error>, + ) -> Harness { + Harness { + serializer, + deserializer, + } + } + + pub fn get_serializer(&self) -> fn(&dyn Any, &mut WriteContext) { + self.serializer + } + + pub fn get_deserializer(&self) -> fn(&mut ReadContext) -> Result, Error> { + self.deserializer + } +} + +pub struct ClassInfo { + type_def: Vec, + type_id: u32, +} + +impl ClassInfo { + pub fn new(fury: &Fury, type_id: u32) -> ClassInfo { + ClassInfo { + type_def: T::type_def(fury), + type_id, + } + } + + pub fn get_type_id(&self) -> u32 { + self.type_id + } + + pub fn get_type_def(&self) -> &Vec { + &self.type_def + } +} + +#[derive(Default)] +pub struct ClassResolver { + serialize_map: HashMap, + type_id_map: HashMap, + class_info_map: HashMap, +} + +impl ClassResolver { + pub fn get_class_info(&self, type_id: TypeId) -> &ClassInfo { + self.class_info_map.get(&type_id).unwrap() + } + + pub fn register(&mut self, class_info: ClassInfo, id: u32) { + fn serializer(this: &dyn Any, context: &mut WriteContext) { + let this = this.downcast_ref::(); + match this { + Some(v) => { + T2::serialize(v, context); + } + None => todo!(), + } + } + + fn deserializer( + context: &mut ReadContext, + ) -> Result, Error> { + match T2::deserialize(context) { + Ok(v) => Ok(Box::new(v)), + Err(e) => Err(e), + } + } + self.type_id_map.insert(TypeId::of::(), id); + self.serialize_map + .insert(id, Harness::new(serializer::, deserializer::)); + self.class_info_map.insert(TypeId::of::(), class_info); + } + + pub fn get_harness_by_type(&self, type_id: TypeId) -> Option<&Harness> { + self.get_harness(*self.type_id_map.get(&type_id).unwrap()) + } + + pub fn get_harness(&self, id: u32) -> Option<&Harness> { + self.serialize_map.get(&id) + } +} diff --git a/rust/fury-core/src/resolver/context.rs b/rust/fury-core/src/resolver/context.rs index ab24ed417f..635d161d92 100644 --- a/rust/fury-core/src/resolver/context.rs +++ b/rust/fury-core/src/resolver/context.rs @@ -41,8 +41,8 @@ impl<'se> WriteContext<'se> { } } - pub fn push_meta(&mut self, type_id: TypeId, type_def: &'static [u8]) -> usize { - self.meta_resolver.push(type_id, type_def) + pub fn push_meta(&mut self, type_id: TypeId) -> usize { + self.meta_resolver.push(type_id, self.fury) } pub fn write_meta(&mut self, offset: usize) { diff --git a/rust/fury-core/src/resolver/meta_resolver.rs b/rust/fury-core/src/resolver/meta_resolver.rs index d39eea6e4c..b9a442c54e 100644 --- a/rust/fury-core/src/resolver/meta_resolver.rs +++ b/rust/fury-core/src/resolver/meta_resolver.rs @@ -17,6 +17,7 @@ use crate::buffer::{Reader, Writer}; use crate::error::Error; +use crate::fury::Fury; use crate::meta::TypeMeta; use std::any::TypeId; use std::collections::HashMap; @@ -44,17 +45,21 @@ impl MetaReaderResolver { #[derive(Default)] pub struct MetaWriterResolver<'a> { - type_defs: Vec<&'a [u8]>, + type_defs: Vec<&'a Vec>, type_id_index_map: HashMap, } #[allow(dead_code)] impl<'a> MetaWriterResolver<'a> { - pub fn push<'b: 'a>(&mut self, type_id: TypeId, type_meta_bytes: &'b [u8]) -> usize { + pub fn push<'b: 'a>(&mut self, type_id: TypeId, fury: &'a Fury) -> usize { match self.type_id_index_map.get(&type_id) { None => { let index = self.type_defs.len(); - self.type_defs.push(type_meta_bytes); + self.type_defs.push( + fury.get_class_resolver() + .get_class_info(type_id) + .get_type_def(), + ); self.type_id_index_map.insert(type_id, index); index } diff --git a/rust/fury-core/src/resolver/mod.rs b/rust/fury-core/src/resolver/mod.rs index ac5114008a..49b3890d45 100644 --- a/rust/fury-core/src/resolver/mod.rs +++ b/rust/fury-core/src/resolver/mod.rs @@ -15,5 +15,6 @@ // specific language governing permissions and limitations // under the License. +pub mod class_resolver; pub mod context; pub mod meta_resolver; diff --git a/rust/fury-core/src/serializer/any.rs b/rust/fury-core/src/serializer/any.rs new file mode 100644 index 0000000000..f05c435c09 --- /dev/null +++ b/rust/fury-core/src/serializer/any.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::error::Error; +use crate::fury::Fury; +use crate::resolver::context::{ReadContext, WriteContext}; +use crate::serializer::Serializer; +use crate::types::{FieldType, Mode, RefFlag}; +use std::any::Any; + +impl Serializer for Box { + fn reserved_space() -> usize { + 0 + } + + fn write(&self, _context: &mut WriteContext) { + panic!("unreachable") + } + + fn read(_context: &mut ReadContext) -> Result { + panic!("unreachable") + } + + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::FuryTypeTag.into() + } + + fn serialize(&self, context: &mut WriteContext) { + context + .get_fury() + .get_class_resolver() + .get_harness_by_type(self.as_ref().type_id()) + .unwrap() + .get_serializer()(self.as_ref(), context); + } + + fn deserialize(context: &mut ReadContext) -> Result { + let reset_cursor = context.reader.reset_cursor_to_here(); + // ref flag + let ref_flag = context.reader.i8(); + + if ref_flag == (RefFlag::NotNullValue as i8) || ref_flag == (RefFlag::RefValue as i8) { + if context.get_fury().get_mode().eq(&Mode::Compatible) { + let meta_index = context.reader.i16(); + let type_id = context.meta_resolver.get(meta_index as usize).get_type_id(); + reset_cursor(&mut context.reader); + context + .get_fury() + .get_class_resolver() + .get_harness(type_id) + .unwrap() + .get_deserializer()(context) + } else { + let type_id = context.reader.i16(); + reset_cursor(&mut context.reader); + context + .get_fury() + .get_class_resolver() + .get_harness(type_id as u32) + .unwrap() + .get_deserializer()(context) + } + } else if ref_flag == (RefFlag::Null as i8) { + Err(Error::Null) + } else if ref_flag == (RefFlag::Ref as i8) { + reset_cursor(&mut context.reader); + Err(Error::Ref) + } else { + reset_cursor(&mut context.reader); + Err(Error::BadRefFlag) + } + } +} diff --git a/rust/fury-core/src/serializer/bool.rs b/rust/fury-core/src/serializer/bool.rs index e3e631d365..b3a98a615f 100644 --- a/rust/fury-core/src/serializer/bool.rs +++ b/rust/fury-core/src/serializer/bool.rs @@ -16,6 +16,7 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; @@ -35,7 +36,7 @@ impl Serializer for bool { Ok(context.reader.u8() == 1) } - fn ty() -> FieldType { - FieldType::BOOL + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::BOOL.into() } } diff --git a/rust/fury-core/src/serializer/datetime.rs b/rust/fury-core/src/serializer/datetime.rs index a1f42d7280..f6bfa736cf 100644 --- a/rust/fury-core/src/serializer/datetime.rs +++ b/rust/fury-core/src/serializer/datetime.rs @@ -16,6 +16,7 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; @@ -42,8 +43,8 @@ impl Serializer for NaiveDateTime { mem::size_of::() } - fn ty() -> FieldType { - FieldType::TIMESTAMP + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::TIMESTAMP.into() } } @@ -67,8 +68,8 @@ impl Serializer for NaiveDate { } } - fn ty() -> FieldType { - FieldType::DATE + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::DATE.into() } } diff --git a/rust/fury-core/src/serializer/list.rs b/rust/fury-core/src/serializer/list.rs index 233fe93fc6..0b147f4379 100644 --- a/rust/fury-core/src/serializer/list.rs +++ b/rust/fury-core/src/serializer/list.rs @@ -16,6 +16,7 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; @@ -52,8 +53,8 @@ where mem::size_of::() } - fn ty() -> FieldType { - FieldType::ARRAY + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::ARRAY.into() } } diff --git a/rust/fury-core/src/serializer/map.rs b/rust/fury-core/src/serializer/map.rs index d324a5827d..e79740f403 100644 --- a/rust/fury-core/src/serializer/map.rs +++ b/rust/fury-core/src/serializer/map.rs @@ -16,6 +16,7 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; @@ -58,8 +59,8 @@ impl Serializer for HashM mem::size_of::() } - fn ty() -> FieldType { - FieldType::MAP + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::MAP.into() } } diff --git a/rust/fury-core/src/serializer/mod.rs b/rust/fury-core/src/serializer/mod.rs index 8a92c67b2b..1803bdfb54 100644 --- a/rust/fury-core/src/serializer/mod.rs +++ b/rust/fury-core/src/serializer/mod.rs @@ -16,9 +16,11 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::{ReadContext, WriteContext}; -use crate::types::{FieldType, RefFlag}; +use crate::types::RefFlag; +mod any; mod bool; mod datetime; mod list; @@ -33,7 +35,7 @@ pub fn serialize(this: &T, context: &mut WriteContext) { // ref flag context.writer.i8(RefFlag::NotNullValue as i8); // type - context.writer.i16(T::ty() as i16); + context.writer.i16(T::get_type_id(context.get_fury())); this.write(context); } @@ -42,13 +44,12 @@ pub fn deserialize(context: &mut ReadContext) -> Result let ref_flag = context.reader.i8(); if ref_flag == (RefFlag::NotNullValue as i8) || ref_flag == (RefFlag::RefValue as i8) { - // type_id - let type_id = context.reader.i16(); - let ty = T::ty(); - if type_id != ty as i16 { + let actual_type_id = context.reader.i16(); + let expected_type_id = T::get_type_id(context.get_fury()); + if actual_type_id != expected_type_id { Err(Error::FieldType { - expected: ty, - actial: type_id, + expected: expected_type_id, + actual: actual_type_id, }) } else { Ok(T::read(context)?) @@ -64,7 +65,7 @@ pub fn deserialize(context: &mut ReadContext) -> Result pub trait Serializer where - Self: Default, + Self: Sized, { /// The fixed memory size of the Type. /// Avoid the memory check, which would hurt performance. @@ -87,5 +88,9 @@ where deserialize(context) } - fn ty() -> FieldType; + fn get_type_id(_fury: &Fury) -> i16; +} + +pub trait StructSerializer: Serializer + 'static { + fn type_def(fury: &Fury) -> Vec; } diff --git a/rust/fury-core/src/serializer/number.rs b/rust/fury-core/src/serializer/number.rs index 4412007e69..17a79928d7 100644 --- a/rust/fury-core/src/serializer/number.rs +++ b/rust/fury-core/src/serializer/number.rs @@ -16,17 +16,12 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; use crate::types::{FieldType, FuryGeneralList}; -#[allow(dead_code)] -fn to_u8_slice(slice: &[T]) -> &[u8] { - let byte_len = std::mem::size_of_val(slice); - unsafe { std::slice::from_raw_parts(slice.as_ptr().cast::(), byte_len) } -} - macro_rules! impl_num_serializer { ($name: ident, $ty:tt, $field_type: expr) => { impl Serializer for $ty { @@ -42,8 +37,8 @@ macro_rules! impl_num_serializer { std::mem::size_of::<$ty>() } - fn ty() -> FieldType { - $field_type + fn get_type_id(_fury: &Fury) -> i16 { + ($field_type).into() } } }; diff --git a/rust/fury-core/src/serializer/option.rs b/rust/fury-core/src/serializer/option.rs index 023297f5a9..dfd607bf4e 100644 --- a/rust/fury-core/src/serializer/option.rs +++ b/rust/fury-core/src/serializer/option.rs @@ -16,10 +16,11 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; -use crate::types::{FieldType, FuryGeneralList, RefFlag}; +use crate::types::{FuryGeneralList, RefFlag}; impl Serializer for Option { fn read(context: &mut ReadContext) -> Result { @@ -32,12 +33,12 @@ impl Serializer for Option { if ref_flag == (RefFlag::NotNullValue as i8) || ref_flag == (RefFlag::RefValue as i8) { // type_id - let type_id = context.reader.i16(); - - if type_id != T::ty() as i16 { + let actual_type_id = context.reader.i16(); + let expected_type_id = T::get_type_id(context.get_fury()); + if actual_type_id != expected_type_id { Err(Error::FieldType { - expected: T::ty(), - actial: type_id, + expected: expected_type_id, + actual: actual_type_id, }) } else { Ok(Some(T::read(context)?)) @@ -65,7 +66,7 @@ impl Serializer for Option { // ref flag context.writer.i8(RefFlag::NotNullValue as i8); // type - context.writer.i16(T::ty() as i16); + context.writer.i16(T::get_type_id(context.get_fury())); v.write(context); } @@ -79,8 +80,8 @@ impl Serializer for Option { std::mem::size_of::() } - fn ty() -> FieldType { - T::ty() + fn get_type_id(fury: &Fury) -> i16 { + T::get_type_id(fury) } } diff --git a/rust/fury-core/src/serializer/primitive_list.rs b/rust/fury-core/src/serializer/primitive_list.rs index b033bb4206..cac36f1c5e 100644 --- a/rust/fury-core/src/serializer/primitive_list.rs +++ b/rust/fury-core/src/serializer/primitive_list.rs @@ -16,6 +16,7 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; @@ -27,11 +28,6 @@ pub fn to_u8_slice(slice: &[T]) -> &[u8] { unsafe { std::slice::from_raw_parts(slice.as_ptr().cast::(), byte_len) } } -fn from_u8_slice(slice: &[u8]) -> Vec { - let byte_len = slice.len() / mem::size_of::(); - unsafe { std::slice::from_raw_parts(slice.as_ptr().cast::(), byte_len) }.to_vec() -} - macro_rules! impl_primitive_vec { ($name: ident, $ty:tt, $field_type: expr) => { impl Serializer for Vec<$ty> { @@ -43,16 +39,29 @@ macro_rules! impl_primitive_vec { fn read(context: &mut ReadContext) -> Result { // length - let len = (context.reader.var_int32() as usize) * mem::size_of::<$ty>(); - Ok(from_u8_slice::<$ty>(context.reader.bytes(len as usize))) + let len = (context.reader.var_int32() as usize); + let is_aligned = context.reader.aligned::<$ty>(); + if is_aligned { + let slice = context.reader.bytes(len * mem::size_of::<$ty>()); + Ok( + unsafe { std::slice::from_raw_parts(slice.as_ptr().cast::<$ty>(), len) } + .to_vec(), + ) + } else { + let mut result = Vec::with_capacity(len); + for _i in 0..len { + result.push(context.reader.$name()); + } + Ok(result) + } } fn reserved_space() -> usize { mem::size_of::() } - fn ty() -> FieldType { - $field_type + fn get_type_id(_fury: &Fury) -> i16 { + ($field_type).into() } } }; @@ -68,8 +77,8 @@ impl Serializer for Vec { mem::size_of::() } - fn ty() -> FieldType { - FieldType::FuryPrimitiveBoolArray + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::FuryPrimitiveBoolArray.into() } fn read(context: &mut ReadContext) -> Result { diff --git a/rust/fury-core/src/serializer/set.rs b/rust/fury-core/src/serializer/set.rs index a652dbd302..4b8b84dba4 100644 --- a/rust/fury-core/src/serializer/set.rs +++ b/rust/fury-core/src/serializer/set.rs @@ -16,6 +16,7 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; @@ -53,8 +54,8 @@ impl Serializer for HashSet { mem::size_of::() } - fn ty() -> FieldType { - FieldType::FurySet + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::FurySet.into() } } diff --git a/rust/fury-core/src/serializer/string.rs b/rust/fury-core/src/serializer/string.rs index 64190fa21e..7f2c1bf39c 100644 --- a/rust/fury-core/src/serializer/string.rs +++ b/rust/fury-core/src/serializer/string.rs @@ -16,6 +16,7 @@ // under the License. use crate::error::Error; +use crate::fury::Fury; use crate::resolver::context::ReadContext; use crate::resolver::context::WriteContext; use crate::serializer::Serializer; @@ -37,8 +38,8 @@ impl Serializer for String { Ok(context.reader.string(len as usize)) } - fn ty() -> FieldType { - FieldType::STRING + fn get_type_id(_fury: &Fury) -> i16 { + FieldType::STRING.into() } } diff --git a/rust/fury-core/src/types.rs b/rust/fury-core/src/types.rs index c1ba2b1f3e..12e43af199 100644 --- a/rust/fury-core/src/types.rs +++ b/rust/fury-core/src/types.rs @@ -16,7 +16,7 @@ // under the License. use crate::error::Error; -use num_enum::TryFromPrimitive; +use num_enum::{IntoPrimitive, TryFromPrimitive}; use std::mem; #[allow(dead_code)] @@ -38,7 +38,7 @@ pub enum RefFlag { RefValue = 0, } -#[derive(Clone, Copy, Debug, PartialEq, TryFromPrimitive)] +#[derive(Clone, Copy, Debug, PartialEq, TryFromPrimitive, IntoPrimitive)] #[repr(i16)] pub enum FieldType { BOOL = 1, @@ -124,8 +124,6 @@ pub fn compute_struct_hash(props: Vec<(&str, FieldType)>) -> u32 { hash } -// todo: flag check -#[allow(dead_code)] pub mod config_flags { pub const IS_NULL_FLAG: u8 = 1 << 0; pub const IS_LITTLE_ENDIAN_FLAG: u8 = 2; diff --git a/rust/fury-derive/src/lib.rs b/rust/fury-derive/src/lib.rs index a9f2605b19..0d8ed47e45 100644 --- a/rust/fury-derive/src/lib.rs +++ b/rust/fury-derive/src/lib.rs @@ -23,22 +23,10 @@ mod fury_row; mod object; mod util; -#[proc_macro_derive(Fury, attributes(tag))] +#[proc_macro_derive(Fury)] pub fn proc_macro_derive_fury_object(input: proc_macro::TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); - let tag = input - .attrs - .iter() - .find(|attr| attr.path().is_ident("tag")) - .expect("should have tag"); - let expr: syn::ExprLit = tag.parse_args().expect("should tag contain string value"); - let tag = match expr.lit { - syn::Lit::Str(s) => s.value(), - _ => { - panic!("tag should be string") - } - }; - object::derive_serializer(&input, &tag) + object::derive_serializer(&input) } #[proc_macro_derive(FuryRow)] diff --git a/rust/fury-derive/src/object/misc.rs b/rust/fury-derive/src/object/misc.rs index a9416e2d02..0f9aac66cb 100644 --- a/rust/fury-derive/src/object/misc.rs +++ b/rust/fury-derive/src/object/misc.rs @@ -24,7 +24,7 @@ fn hash(fields: &[&Field]) -> TokenStream { let ty = &field.ty; let name = format!("{}", field.ident.as_ref().expect("should be field name")); quote! { - (#name, <#ty as fury_core::serializer::Serializer>::ty()) + (#name, <#ty as fury_core::serializer::Serializer>::get_type_id()) } }); @@ -48,46 +48,31 @@ fn type_def(fields: &[&Field]) -> TokenStream { let ty = &field.ty; let name = format!("{}", field.ident.as_ref().expect("should be field name")); quote! { - fury_core::meta::FieldInfo::new(#name, <#ty as fury_core::serializer::Serializer>::ty()) + fury_core::meta::FieldInfo::new(#name, <#ty as fury_core::serializer::Serializer>::get_type_id(fury)) } }); quote! { - fn fury_type_def() -> &'static [u8] { - use std::sync::Once; - static mut type_definition: Vec = Vec::new(); - static type_definition_once: Once = Once::new(); - unsafe { - type_definition_once.call_once(|| { - type_definition = fury_core::meta::TypeMeta::from_fields( - 0, - vec![#(#field_infos),*] - ).to_bytes().unwrap(); - }); - type_definition.as_slice() - } + fn type_def(fury: &fury_core::fury::Fury) -> Vec { + fury_core::meta::TypeMeta::from_fields( + 0, + vec![#(#field_infos),*] + ).to_bytes().unwrap() } } } -pub fn gen_in_struct_impl(fields: &[&Field], tag: &String) -> TokenStream { - let hash_token_stream = hash(fields); +pub fn gen_in_struct_impl(fields: &[&Field]) -> TokenStream { + let _hash_token_stream = hash(fields); let type_def_token_stream = type_def(fields); quote! { - #hash_token_stream - #type_def_token_stream - - fn fury_tag() -> &'static str { - #tag - } } } - pub fn gen() -> TokenStream { quote! { - fn ty() -> fury_core::types::FieldType { - fury_core::types::FieldType::FuryTypeTag + fn get_type_id(fury: &fury_core::fury::Fury) -> i16 { + fury.get_class_resolver().get_class_info(std::any::TypeId::of::()).get_type_id() as i16 } } } diff --git a/rust/fury-derive/src/object/read.rs b/rust/fury-derive/src/object/read.rs index de7176af7d..7249609a81 100644 --- a/rust/fury-derive/src/object/read.rs +++ b/rust/fury-derive/src/object/read.rs @@ -16,69 +16,87 @@ // under the License. use proc_macro2::{Ident, TokenStream}; -use quote::quote; +use quote::{format_ident, quote}; use syn::Field; -fn read(name: &Ident, fields: &[&Field]) -> TokenStream { +fn create_private_field_name(field: &Field) -> Ident { + format_ident!("_{}", field.ident.as_ref().expect("")) +} + +fn bind(fields: &[&Field]) -> Vec { + fields + .iter() + .map(|field| { + let ty = &field.ty; + let var_name = create_private_field_name(field); + quote! { + let mut #var_name: Option<#ty> = None; + } + }) + .collect() +} + +fn create(fields: &[&Field]) -> Vec { + fields + .iter() + .map(|field| { + let name = &field.ident; + let var_name = create_private_field_name(field); + quote! { + #name: #var_name.unwrap() + } + }) + .collect() +} + +fn read(fields: &[&Field]) -> TokenStream { let assign_stmt = fields.iter().map(|field| { let ty = &field.ty; - let ident = &field.ident; + let name = &field.ident; quote! { - #ident: <#ty as fury_core::serializer::Serializer>::deserialize(context)? + #name: <#ty as fury_core::serializer::Serializer>::deserialize(context)? } }); + quote! { fn read(context: &mut fury_core::resolver::context::ReadContext) -> Result { - // read tag string - context.read_tag()?; - // read tag hash - let hash = context.reader.u32(); - let expected = #name::fury_hash(); - if(hash != expected) { - Err(fury_core::error::Error::StructHash{ expected, actial: hash }) - } else { - Ok(Self { - #(#assign_stmt),* - }) - } + Ok(Self { + #(#assign_stmt),* + }) } } } -fn deserialize_compatible(name: &Ident, fields: &[&Field]) -> TokenStream { +fn deserialize_compatible(fields: &[&Field]) -> TokenStream { let pattern_item = fields.iter().enumerate().map(|(index, field)| { let ty = &field.ty; - let name = &field.ident; + let var_name = create_private_field_name(field); quote! { #index => { - result.#name = <#ty as fury_core::serializer::Serializer>::deserialize(context)? + #var_name = Some(<#ty as fury_core::serializer::Serializer>::deserialize(context)?); } } }); + let bind: Vec = bind(fields); + let create: Vec = create(fields); quote! { let ref_flag = context.reader.i8(); if ref_flag == (fury_core::types::RefFlag::NotNullValue as i8) || ref_flag == (fury_core::types::RefFlag::RefValue as i8) { - let mut result = Self::default(); let meta_index = context.reader.i16() as usize; let meta = context.get_meta(meta_index).clone(); let fields = meta.get_field_info(); - // read tag string - context.read_tag()?; - // read tag hash - let hash = context.reader.u32(); - let expected = #name::fury_hash(); - if(hash != expected) { - return Err(fury_core::error::Error::StructHash{ expected, actial: hash }) - } + #(#bind)* for (idx, _field_info) in fields.iter().enumerate() { match idx { #(#pattern_item),* _ => { - panic!("not implement yet") + panic!("not implement yet"); } } } - Ok(result) + Ok(Self { + #(#create),* + }) } else if ref_flag == (fury_core::types::RefFlag::Null as i8) { Err(fury_core::error::Error::Null) } else if ref_flag == (fury_core::types::RefFlag::Ref as i8) { @@ -89,9 +107,10 @@ fn deserialize_compatible(name: &Ident, fields: &[&Field]) -> TokenStream { } } -pub fn gen(name: &Ident, fields: &[&Field]) -> TokenStream { - let read_token_stream = read(name, fields); - let compatible_token_stream = deserialize_compatible(name, fields); +pub fn gen(fields: &[&Field]) -> TokenStream { + let read_token_stream = read(fields); + let compatible_token_stream = deserialize_compatible(fields); + quote! { fn deserialize(context: &mut fury_core::resolver::context::ReadContext) -> Result { match context.get_fury().get_mode() { diff --git a/rust/fury-derive/src/object/serializer.rs b/rust/fury-derive/src/object/serializer.rs index c8cbd721b9..c87ad0ddb1 100644 --- a/rust/fury-derive/src/object/serializer.rs +++ b/rust/fury-derive/src/object/serializer.rs @@ -20,7 +20,7 @@ use crate::util::sorted_fields; use proc_macro::TokenStream; use quote::quote; -pub fn derive_serializer(ast: &syn::DeriveInput, tag: &String) -> TokenStream { +pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream { let name = &ast.ident; let fields = match &ast.data { syn::Data::Struct(s) => sorted_fields(&s.fields), @@ -30,12 +30,12 @@ pub fn derive_serializer(ast: &syn::DeriveInput, tag: &String) -> TokenStream { }; let misc_token_stream = misc::gen(); - let struct_impl_token_stream = misc::gen_in_struct_impl(&fields, tag); - let write_token_stream = write::gen(name, &fields); - let read_token_stream = read::gen(name, &fields); + let struct_impl_token_stream = misc::gen_in_struct_impl(&fields); + let write_token_stream = write::gen(&fields); + let read_token_stream = read::gen(&fields); let gen = quote! { - impl #name { + impl fury_core::serializer::StructSerializer for #name { #struct_impl_token_stream } impl fury_core::types::FuryGeneralList for #name {} diff --git a/rust/fury-derive/src/object/write.rs b/rust/fury-derive/src/object/write.rs index 83236be727..1a3bd2f52d 100644 --- a/rust/fury-derive/src/object/write.rs +++ b/rust/fury-derive/src/object/write.rs @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -use proc_macro2::{Ident, TokenStream}; +use proc_macro2::TokenStream; use quote::quote; use syn::Field; -pub fn gen(name: &Ident, fields: &[&Field]) -> TokenStream { +pub fn gen(fields: &[&Field]) -> TokenStream { let accessor_expr = fields.iter().map(|field| { let ty = &field.ty; let ident = &field.ident; @@ -36,8 +36,6 @@ pub fn gen(name: &Ident, fields: &[&Field]) -> TokenStream { } }); - let tag_byte_len = format!("{}", name).len(); - quote! { fn serialize(&self, context: &mut fury_core::resolver::context::WriteContext) { match context.get_fury().get_mode() { @@ -47,8 +45,7 @@ pub fn gen(name: &Ident, fields: &[&Field]) -> TokenStream { fury_core::types::Mode::Compatible => { context.writer.i8(fury_core::types::RefFlag::NotNullValue as i8); let meta_index = context.push_meta( - std::any::TypeId::of::(), - #name::fury_type_def() + std::any::TypeId::of::() ) as i16; context.writer.i16(meta_index); self.write(context); @@ -58,17 +55,12 @@ pub fn gen(name: &Ident, fields: &[&Field]) -> TokenStream { fn write(&self, context: &mut fury_core::resolver::context::WriteContext) { - // write tag string - context.write_tag(#name::fury_tag()); - // write tag hash - context.writer.u32(#name::fury_hash()); // write fields #(#accessor_expr)* } fn reserved_space() -> usize { - // struct have four byte hash - #tag_byte_len + 4 + #(#reserved_size_expr)+* + #(#reserved_size_expr)+* } } } diff --git a/rust/tests/tests/test_complex_struct.rs b/rust/tests/tests/test_complex_struct.rs index 4eea7a9ec7..0406d170ca 100644 --- a/rust/tests/tests/test_complex_struct.rs +++ b/rust/tests/tests/test_complex_struct.rs @@ -19,18 +19,43 @@ use chrono::{DateTime, NaiveDate, NaiveDateTime}; use fury_core::fury::Fury; use fury_core::types::Mode; use fury_derive::Fury; +use std::any::Any; use std::collections::HashMap; +#[test] +fn any() { + #[derive(Fury, Debug)] + struct Animal { + f3: String, + } + + #[derive(Fury, Debug)] + struct Person { + f1: Box, + } + + let person = Person { + f1: Box::new(Animal { + f3: String::from("hello"), + }), + }; + + let mut fury = Fury::default(); + fury.register::(999); + fury.register::(1000); + let bin = fury.serialize(&person); + let obj: Person = fury.deserialize(&bin).expect(""); + assert!(obj.f1.is::()) +} + #[test] fn complex_struct() { #[derive(Fury, Debug, PartialEq, Default)] - #[tag("example.foo2")] struct Animal { category: String, } #[derive(Fury, Debug, PartialEq, Default)] - #[tag("example.foo")] struct Person { c1: Vec, // binary c2: Vec, // primitive array @@ -66,7 +91,10 @@ fn complex_struct() { c5: 2.0, c6: 4.0, }; - let fury = Fury::default().mode(Mode::Compatible); + let mut fury = Fury::default().mode(Mode::Compatible); + fury.register::(999); + fury.register::(899); + let bin: Vec = fury.serialize(&person); let obj: Person = fury.deserialize(&bin).expect("should success"); assert_eq!(person, obj); @@ -75,13 +103,11 @@ fn complex_struct() { #[test] fn encode_to_obin() { #[derive(Fury, Debug, PartialEq, Default)] - #[tag("example.foo2")] struct Animal { category: String, } #[derive(Fury, Debug, PartialEq, Default)] - #[tag("example.ComplexObject")] struct Person { f1: String, f2: HashMap, @@ -93,7 +119,9 @@ fn encode_to_obin() { f8: f64, f10: HashMap, } - let fury = Fury::default(); + let mut fury = Fury::default(); + fury.register::(999); + fury.register::(899); let bin: Vec = fury.serialize(&Person { f1: "Hello".to_string(), f2: HashMap::from([("hello1".to_string(), 1), ("hello2".to_string(), 2)]),